2026-02-22T22:20:16.502 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-22T22:20:16.520 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T22:20:16.605 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215 branch: wip-yuri6-testing-2026-02-22-1629-squid description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-ha-workunit} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '64215' kernel: branch: distro kdb: true sha1: distro ktype: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run the rbd_mirror_ha.sh workunit to test the rbd-mirror daemon name: yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: wip-yuri6-testing-2026-02-22-1629-squid ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: none client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.4: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.5: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.6: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bluefs allocator: hybrid bluestore allocator: hybrid bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-yuri6-testing-2026-02-22-1629-squid sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 9196 sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-yuri6-testing-2026-02-22-1629-squid suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 targets: trial103.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJStKL36rec08+9wIWCxUicGZX4AdrDb46i8HomuX0QVr2fL97guGi9kcXpCaGJ4WhA3TAw3hAIN+nvpUOYsYcs= trial190.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEyAAOy4lMO91njfkj1AgusrY18FeVwgcFvwyeLtBVYRVORdeIGWSybVFvdCKSY06oL98dZ4st4HimDYzbkhVkk= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_ha.sh env: CEPH_ARGS: '' RBD_MIRROR_USE_EXISTING_CLUSTER: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-22_19:35:38 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653 2026-02-22T22:20:16.605 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa; will attempt to use it 2026-02-22T22:20:16.605 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa/tasks 2026-02-22T22:20:16.606 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-22T22:20:16.607 INFO:teuthology.task.internal:Checking packages... 2026-02-22T22:20:16.607 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'f90f4eaebb28b619cf76e80cbbe997656da69579' 2026-02-22T22:20:16.607 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-22T22:20:16.607 INFO:teuthology.packaging:ref: None 2026-02-22T22:20:16.607 INFO:teuthology.packaging:tag: None 2026-02-22T22:20:16.607 INFO:teuthology.packaging:branch: wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:20:16.607 INFO:teuthology.packaging:sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:20:16.607 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:20:16.726 INFO:teuthology.task.internal:Found packages for ceph version 19.2.3-689-gf90f4eae-1jammy 2026-02-22T22:20:16.728 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-22T22:20:16.739 INFO:teuthology.task.internal:no buildpackages task found 2026-02-22T22:20:16.739 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-22T22:20:16.752 INFO:teuthology.task.internal:Saving configuration 2026-02-22T22:20:16.762 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-22T22:20:16.773 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-22T22:20:16.838 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial103.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 22:16:15.971187', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ8ane14QK4xf+zz3413DttuRj+kGdZCdhhO6TWB45Qs'} 2026-02-22T22:20:16.899 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial190.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 22:16:15.970581', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICY+C0m66XAu7MgFCFNTVIFMhwMpHH1t0cYZynvE+QyI'} 2026-02-22T22:20:16.899 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-22T22:20:16.909 INFO:teuthology.task.internal:roles: ubuntu@trial103.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-22T22:20:16.909 INFO:teuthology.task.internal:roles: ubuntu@trial190.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-22T22:20:16.909 INFO:teuthology.run_tasks:Running task console_log... 2026-02-22T22:20:17.061 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7353095f7e20>, signals=[15]) 2026-02-22T22:20:17.061 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-22T22:20:17.071 INFO:teuthology.task.internal:Opening connections... 2026-02-22T22:20:17.071 DEBUG:teuthology.task.internal:connecting to ubuntu@trial103.front.sepia.ceph.com 2026-02-22T22:20:17.072 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial103.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:20:17.135 DEBUG:teuthology.task.internal:connecting to ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:20:17.136 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial190.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:20:17.201 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-22T22:20:17.213 DEBUG:teuthology.orchestra.run.trial103:> uname -m 2026-02-22T22:20:17.216 INFO:teuthology.orchestra.run.trial103.stdout:x86_64 2026-02-22T22:20:17.217 DEBUG:teuthology.orchestra.run.trial103:> cat /etc/os-release 2026-02-22T22:20:17.262 INFO:teuthology.orchestra.run.trial103.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-02-22T22:20:17.262 INFO:teuthology.orchestra.run.trial103.stdout:NAME="Ubuntu" 2026-02-22T22:20:17.262 INFO:teuthology.orchestra.run.trial103.stdout:VERSION_ID="22.04" 2026-02-22T22:20:17.262 INFO:teuthology.orchestra.run.trial103.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-02-22T22:20:17.262 INFO:teuthology.orchestra.run.trial103.stdout:VERSION_CODENAME=jammy 2026-02-22T22:20:17.262 INFO:teuthology.orchestra.run.trial103.stdout:ID=ubuntu 2026-02-22T22:20:17.262 INFO:teuthology.orchestra.run.trial103.stdout:ID_LIKE=debian 2026-02-22T22:20:17.263 INFO:teuthology.orchestra.run.trial103.stdout:HOME_URL="https://www.ubuntu.com/" 2026-02-22T22:20:17.263 INFO:teuthology.orchestra.run.trial103.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-02-22T22:20:17.263 INFO:teuthology.orchestra.run.trial103.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-02-22T22:20:17.263 INFO:teuthology.orchestra.run.trial103.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-02-22T22:20:17.263 INFO:teuthology.orchestra.run.trial103.stdout:UBUNTU_CODENAME=jammy 2026-02-22T22:20:17.263 INFO:teuthology.lock.ops:Updating trial103.front.sepia.ceph.com on lock server 2026-02-22T22:20:17.325 DEBUG:teuthology.orchestra.run.trial190:> uname -m 2026-02-22T22:20:17.329 INFO:teuthology.orchestra.run.trial190.stdout:x86_64 2026-02-22T22:20:17.329 DEBUG:teuthology.orchestra.run.trial190:> cat /etc/os-release 2026-02-22T22:20:17.373 INFO:teuthology.orchestra.run.trial190.stdout:PRETTY_NAME="Ubuntu 22.04.5 LTS" 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:NAME="Ubuntu" 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:VERSION_ID="22.04" 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:VERSION="22.04.5 LTS (Jammy Jellyfish)" 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:VERSION_CODENAME=jammy 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:ID=ubuntu 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:ID_LIKE=debian 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:HOME_URL="https://www.ubuntu.com/" 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2026-02-22T22:20:17.374 INFO:teuthology.orchestra.run.trial190.stdout:UBUNTU_CODENAME=jammy 2026-02-22T22:20:17.374 INFO:teuthology.lock.ops:Updating trial190.front.sepia.ceph.com on lock server 2026-02-22T22:20:17.443 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-22T22:20:17.456 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-22T22:20:17.467 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-22T22:20:17.467 DEBUG:teuthology.orchestra.run.trial103:> test '!' -e /home/ubuntu/cephtest 2026-02-22T22:20:17.468 DEBUG:teuthology.orchestra.run.trial190:> test '!' -e /home/ubuntu/cephtest 2026-02-22T22:20:17.472 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-22T22:20:17.482 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-22T22:20:17.482 DEBUG:teuthology.orchestra.run.trial103:> test -z $(ls -A /var/lib/ceph) 2026-02-22T22:20:17.511 DEBUG:teuthology.orchestra.run.trial190:> test -z $(ls -A /var/lib/ceph) 2026-02-22T22:20:17.514 INFO:teuthology.orchestra.run.trial103.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-02-22T22:20:17.518 INFO:teuthology.orchestra.run.trial190.stderr:ls: cannot access '/var/lib/ceph': No such file or directory 2026-02-22T22:20:17.519 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-22T22:20:17.646 INFO:teuthology.run_tasks:Running task kernel... 2026-02-22T22:20:17.665 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-22T22:20:17.665 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-22T22:20:17.665 DEBUG:teuthology.orchestra.run.trial103:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T22:20:17.666 DEBUG:teuthology.orchestra.run.trial190:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T22:20:17.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:20:17.668 DEBUG:teuthology.orchestra.run.trial190:> uname -r 2026-02-22T22:20:17.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:20:17.669 DEBUG:teuthology.orchestra.run.trial103:> uname -r 2026-02-22T22:20:17.713 INFO:teuthology.orchestra.run.trial190.stdout:5.15.0-170-generic 2026-02-22T22:20:17.713 INFO:teuthology.task.kernel:Running kernel on trial190: 5.15.0-170-generic 2026-02-22T22:20:17.713 DEBUG:teuthology.orchestra.run.trial190:> sudo apt-get clean 2026-02-22T22:20:17.714 INFO:teuthology.orchestra.run.trial103.stdout:5.15.0-170-generic 2026-02-22T22:20:17.714 INFO:teuthology.task.kernel:Running kernel on trial103: 5.15.0-170-generic 2026-02-22T22:20:17.714 DEBUG:teuthology.orchestra.run.trial103:> sudo apt-get clean 2026-02-22T22:20:17.771 DEBUG:teuthology.orchestra.run.trial190:> sudo apt-get update 2026-02-22T22:20:17.773 DEBUG:teuthology.orchestra.run.trial103:> sudo apt-get update 2026-02-22T22:20:18.278 INFO:teuthology.orchestra.run.trial103.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-22T22:20:18.288 INFO:teuthology.orchestra.run.trial190.stdout:Hit:1 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-22T22:20:18.368 INFO:teuthology.orchestra.run.trial103.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2026-02-22T22:20:18.381 INFO:teuthology.orchestra.run.trial190.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2026-02-22T22:20:18.704 INFO:teuthology.orchestra.run.trial103.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2026-02-22T22:20:18.731 INFO:teuthology.orchestra.run.trial190.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2026-02-22T22:20:18.852 INFO:teuthology.orchestra.run.trial103.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2026-02-22T22:20:18.886 INFO:teuthology.orchestra.run.trial190.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2026-02-22T22:20:18.991 INFO:teuthology.orchestra.run.trial103.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [3245 kB] 2026-02-22T22:20:19.030 INFO:teuthology.orchestra.run.trial190.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [3245 kB] 2026-02-22T22:20:19.392 INFO:teuthology.orchestra.run.trial103.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [495 kB] 2026-02-22T22:20:19.405 INFO:teuthology.orchestra.run.trial103.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [19.1 kB] 2026-02-22T22:20:19.406 INFO:teuthology.orchestra.run.trial103.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1256 kB] 2026-02-22T22:20:19.440 INFO:teuthology.orchestra.run.trial103.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [314 kB] 2026-02-22T22:20:19.445 INFO:teuthology.orchestra.run.trial190.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [495 kB] 2026-02-22T22:20:19.448 INFO:teuthology.orchestra.run.trial103.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [30.3 kB] 2026-02-22T22:20:19.448 INFO:teuthology.orchestra.run.trial103.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [5310 kB] 2026-02-22T22:20:19.457 INFO:teuthology.orchestra.run.trial190.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [19.1 kB] 2026-02-22T22:20:19.458 INFO:teuthology.orchestra.run.trial190.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1256 kB] 2026-02-22T22:20:19.494 INFO:teuthology.orchestra.run.trial190.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [314 kB] 2026-02-22T22:20:19.502 INFO:teuthology.orchestra.run.trial190.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [30.3 kB] 2026-02-22T22:20:19.502 INFO:teuthology.orchestra.run.trial190.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [5310 kB] 2026-02-22T22:20:19.605 INFO:teuthology.orchestra.run.trial103.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [1005 kB] 2026-02-22T22:20:19.628 INFO:teuthology.orchestra.run.trial103.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2983 kB] 2026-02-22T22:20:19.665 INFO:teuthology.orchestra.run.trial190.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [1005 kB] 2026-02-22T22:20:19.687 INFO:teuthology.orchestra.run.trial190.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 Packages [2983 kB] 2026-02-22T22:20:19.715 INFO:teuthology.orchestra.run.trial103.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-security/main Translation-en [427 kB] 2026-02-22T22:20:19.725 INFO:teuthology.orchestra.run.trial103.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [14.1 kB] 2026-02-22T22:20:19.725 INFO:teuthology.orchestra.run.trial103.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1018 kB] 2026-02-22T22:20:19.760 INFO:teuthology.orchestra.run.trial103.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [22.7 kB] 2026-02-22T22:20:19.760 INFO:teuthology.orchestra.run.trial103.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [5147 kB] 2026-02-22T22:20:19.775 INFO:teuthology.orchestra.run.trial190.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-security/main Translation-en [427 kB] 2026-02-22T22:20:19.787 INFO:teuthology.orchestra.run.trial190.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [14.1 kB] 2026-02-22T22:20:19.787 INFO:teuthology.orchestra.run.trial190.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1018 kB] 2026-02-22T22:20:19.820 INFO:teuthology.orchestra.run.trial190.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [22.7 kB] 2026-02-22T22:20:19.823 INFO:teuthology.orchestra.run.trial190.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [5147 kB] 2026-02-22T22:20:19.905 INFO:teuthology.orchestra.run.trial103.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy-security/restricted Translation-en [979 kB] 2026-02-22T22:20:19.977 INFO:teuthology.orchestra.run.trial190.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy-security/restricted Translation-en [979 kB] 2026-02-22T22:20:23.368 INFO:teuthology.orchestra.run.trial190.stdout:Fetched 22.6 MB in 3s (7688 kB/s) 2026-02-22T22:20:23.390 INFO:teuthology.orchestra.run.trial103.stdout:Fetched 22.6 MB in 3s (7690 kB/s) 2026-02-22T22:20:23.898 INFO:teuthology.orchestra.run.trial190.stdout:Reading package lists... 2026-02-22T22:20:23.910 DEBUG:teuthology.orchestra.run.trial190:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-22T22:20:23.917 INFO:teuthology.orchestra.run.trial103.stdout:Reading package lists... 2026-02-22T22:20:23.930 DEBUG:teuthology.orchestra.run.trial103:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2026-02-22T22:20:23.943 INFO:teuthology.orchestra.run.trial190.stdout:Reading package lists... 2026-02-22T22:20:23.963 INFO:teuthology.orchestra.run.trial103.stdout:Reading package lists... 2026-02-22T22:20:24.054 INFO:teuthology.orchestra.run.trial190.stdout:Building dependency tree... 2026-02-22T22:20:24.054 INFO:teuthology.orchestra.run.trial190.stdout:Reading state information... 2026-02-22T22:20:24.073 INFO:teuthology.orchestra.run.trial103.stdout:Building dependency tree... 2026-02-22T22:20:24.073 INFO:teuthology.orchestra.run.trial103.stdout:Reading state information... 2026-02-22T22:20:24.139 INFO:teuthology.orchestra.run.trial190.stdout:linux-image-generic is already the newest version (5.15.0.170.159). 2026-02-22T22:20:24.140 INFO:teuthology.orchestra.run.trial190.stdout:linux-image-generic set to manually installed. 2026-02-22T22:20:24.140 INFO:teuthology.orchestra.run.trial190.stdout:The following packages were automatically installed and are no longer required: 2026-02-22T22:20:24.140 INFO:teuthology.orchestra.run.trial190.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-22T22:20:24.140 INFO:teuthology.orchestra.run.trial190.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-22T22:20:24.140 INFO:teuthology.orchestra.run.trial190.stdout: sg3-utils-udev 2026-02-22T22:20:24.140 INFO:teuthology.orchestra.run.trial190.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-22T22:20:24.159 INFO:teuthology.orchestra.run.trial190.stdout:0 upgraded, 0 newly installed, 0 to remove and 13 not upgraded. 2026-02-22T22:20:24.160 DEBUG:teuthology.orchestra.run.trial190:> dpkg -s linux-image-generic 2026-02-22T22:20:24.163 INFO:teuthology.orchestra.run.trial103.stdout:linux-image-generic is already the newest version (5.15.0.170.159). 2026-02-22T22:20:24.163 INFO:teuthology.orchestra.run.trial103.stdout:linux-image-generic set to manually installed. 2026-02-22T22:20:24.163 INFO:teuthology.orchestra.run.trial103.stdout:The following packages were automatically installed and are no longer required: 2026-02-22T22:20:24.163 INFO:teuthology.orchestra.run.trial103.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-22T22:20:24.164 INFO:teuthology.orchestra.run.trial103.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-22T22:20:24.164 INFO:teuthology.orchestra.run.trial103.stdout: sg3-utils-udev 2026-02-22T22:20:24.164 INFO:teuthology.orchestra.run.trial103.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-22T22:20:24.183 INFO:teuthology.orchestra.run.trial103.stdout:0 upgraded, 0 newly installed, 0 to remove and 13 not upgraded. 2026-02-22T22:20:24.186 DEBUG:teuthology.orchestra.run.trial103:> dpkg -s linux-image-generic 2026-02-22T22:20:24.212 INFO:teuthology.orchestra.run.trial190.stdout:Package: linux-image-generic 2026-02-22T22:20:24.212 INFO:teuthology.orchestra.run.trial190.stdout:Status: install ok installed 2026-02-22T22:20:24.212 INFO:teuthology.orchestra.run.trial190.stdout:Priority: optional 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout:Section: kernel 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout:Installed-Size: 21 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout:Maintainer: Ubuntu Kernel Team 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout:Architecture: amd64 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout:Source: linux-meta 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout:Version: 5.15.0.170.159 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout:Recommends: thermald 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout:Description: Generic Linux kernel image 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout: This package will always depend on the latest generic kernel image 2026-02-22T22:20:24.213 INFO:teuthology.orchestra.run.trial190.stdout: available. 2026-02-22T22:20:24.214 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T22:20:24.214 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-22T22:20:24.214 DEBUG:teuthology.orchestra.run.trial190:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T22:20:24.236 INFO:teuthology.orchestra.run.trial103.stdout:Package: linux-image-generic 2026-02-22T22:20:24.236 INFO:teuthology.orchestra.run.trial103.stdout:Status: install ok installed 2026-02-22T22:20:24.236 INFO:teuthology.orchestra.run.trial103.stdout:Priority: optional 2026-02-22T22:20:24.236 INFO:teuthology.orchestra.run.trial103.stdout:Section: kernel 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout:Installed-Size: 21 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout:Maintainer: Ubuntu Kernel Team 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout:Architecture: amd64 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout:Source: linux-meta 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout:Version: 5.15.0.170.159 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.6), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-170), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.6) 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout:Depends: linux-image-5.15.0-170-generic, linux-modules-extra-5.15.0-170-generic, linux-firmware, intel-microcode, amd64-microcode 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout:Recommends: thermald 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout:Description: Generic Linux kernel image 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout: This package will always depend on the latest generic kernel image 2026-02-22T22:20:24.237 INFO:teuthology.orchestra.run.trial103.stdout: available. 2026-02-22T22:20:24.238 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T22:20:24.238 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-22T22:20:24.238 DEBUG:teuthology.orchestra.run.trial103:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T22:20:24.262 INFO:teuthology.orchestra.run.trial190.stdout:ttyS1 2026-02-22T22:20:24.290 INFO:teuthology.orchestra.run.trial103.stdout:ttyS1 2026-02-22T22:20:24.304 DEBUG:teuthology.parallel:result is None 2026-02-22T22:20:24.313 DEBUG:teuthology.parallel:result is None 2026-02-22T22:20:24.313 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-22T22:20:24.325 INFO:teuthology.task.internal:Creating test directory... 2026-02-22T22:20:24.326 DEBUG:teuthology.orchestra.run.trial103:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T22:20:24.356 DEBUG:teuthology.orchestra.run.trial190:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T22:20:24.361 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-22T22:20:24.373 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-22T22:20:24.385 INFO:teuthology.task.internal:Creating archive directory... 2026-02-22T22:20:24.385 DEBUG:teuthology.orchestra.run.trial103:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T22:20:24.408 DEBUG:teuthology.orchestra.run.trial190:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T22:20:24.422 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-22T22:20:24.433 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-22T22:20:24.434 DEBUG:teuthology.orchestra.run.trial103:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T22:20:24.456 DEBUG:teuthology.orchestra.run.trial190:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T22:20:24.463 INFO:teuthology.orchestra.run.trial103.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:20:24.466 INFO:teuthology.orchestra.run.trial190.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:20:24.469 INFO:teuthology.orchestra.run.trial103.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:20:24.473 INFO:teuthology.orchestra.run.trial190.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:20:24.474 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-22T22:20:24.485 INFO:teuthology.task.internal:Configuring sudo... 2026-02-22T22:20:24.486 DEBUG:teuthology.orchestra.run.trial103:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T22:20:24.515 DEBUG:teuthology.orchestra.run.trial190:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T22:20:24.523 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-22T22:20:24.535 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-22T22:20:24.535 DEBUG:teuthology.orchestra.run.trial103:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T22:20:24.563 DEBUG:teuthology.orchestra.run.trial190:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T22:20:24.569 DEBUG:teuthology.orchestra.run.trial103:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:20:24.610 DEBUG:teuthology.orchestra.run.trial103:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:20:24.655 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:20:24.655 DEBUG:teuthology.orchestra.run.trial103:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T22:20:24.707 DEBUG:teuthology.orchestra.run.trial190:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:20:24.711 DEBUG:teuthology.orchestra.run.trial190:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:20:24.758 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:20:24.759 DEBUG:teuthology.orchestra.run.trial190:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T22:20:24.812 DEBUG:teuthology.orchestra.run.trial103:> sudo service rsyslog restart 2026-02-22T22:20:24.814 DEBUG:teuthology.orchestra.run.trial190:> sudo service rsyslog restart 2026-02-22T22:20:24.872 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-22T22:20:24.887 INFO:teuthology.task.internal:Starting timer... 2026-02-22T22:20:24.888 INFO:teuthology.run_tasks:Running task pcp... 2026-02-22T22:20:24.908 INFO:teuthology.run_tasks:Running task selinux... 2026-02-22T22:20:24.926 DEBUG:teuthology.task.selinux:Excluding trial103: OS 'ubuntu' does not support SELinux 2026-02-22T22:20:24.927 DEBUG:teuthology.task.selinux:Excluding trial190: OS 'ubuntu' does not support SELinux 2026-02-22T22:20:24.927 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-22T22:20:24.927 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2026-02-22T22:20:24.927 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-22T22:20:24.927 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-22T22:20:24.942 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-22T22:20:24.951 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-22T22:20:24.952 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial103.front.sepia.ceph.com,trial190.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-22T22:22:49.025 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial103.front.sepia.ceph.com'), Remote(name='ubuntu@trial190.front.sepia.ceph.com')] 2026-02-22T22:22:49.033 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial103.front.sepia.ceph.com' 2026-02-22T22:22:49.034 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial103.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:22:49.143 DEBUG:teuthology.orchestra.run.trial103:> true 2026-02-22T22:22:49.395 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial103.front.sepia.ceph.com' 2026-02-22T22:22:49.396 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial190.front.sepia.ceph.com' 2026-02-22T22:22:49.397 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial190.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:22:49.484 DEBUG:teuthology.orchestra.run.trial190:> true 2026-02-22T22:22:49.727 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial190.front.sepia.ceph.com' 2026-02-22T22:22:49.727 INFO:teuthology.run_tasks:Running task clock... 2026-02-22T22:22:49.745 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-22T22:22:49.746 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T22:22:49.746 DEBUG:teuthology.orchestra.run.trial103:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:22:49.749 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T22:22:49.749 DEBUG:teuthology.orchestra.run.trial190:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:22:49.766 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-02-22T22:22:49.766 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: Command line: ntpd -gq 2026-02-22T22:22:49.766 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: ---------------------------------------------------- 2026-02-22T22:22:49.766 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: ntp-4 is maintained by Network Time Foundation, 2026-02-22T22:22:49.766 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-02-22T22:22:49.766 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: corporation. Support and training for ntp-4 are 2026-02-22T22:22:49.766 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: available at https://www.nwtime.org/support 2026-02-22T22:22:49.766 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: ---------------------------------------------------- 2026-02-22T22:22:49.767 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: proto: precision = 0.050 usec (-24) 2026-02-22T22:22:49.767 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: basedate set to 2022-02-04 2026-02-22T22:22:49.767 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: gps base set to 2022-02-06 (week 2196) 2026-02-22T22:22:49.768 INFO:teuthology.orchestra.run.trial103.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-22T22:22:49.768 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-22T22:22:49.768 INFO:teuthology.orchestra.run.trial103.stderr:restrict ::: KOD does nothing without LIMITED. 2026-02-22T22:22:49.769 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: restrict ::: KOD does nothing without LIMITED. 2026-02-22T22:22:49.770 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: Listen and drop on 0 v6wildcard [::]:123 2026-02-22T22:22:49.770 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-02-22T22:22:49.770 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: Listen normally on 2 lo 127.0.0.1:123 2026-02-22T22:22:49.770 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: Listen normally on 3 enp1s0f0 10.20.193.103:123 2026-02-22T22:22:49.770 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: Listen normally on 4 lo [::1]:123 2026-02-22T22:22:49.770 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: Listen normally on 5 enp1s0f0 [fe80::925a:8ff:fe77:5f7a%2]:123 2026-02-22T22:22:49.770 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:49 ntpd[9421]: Listening on routing socket on fd #22 for interface updates 2026-02-22T22:22:49.790 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2026-02-22T22:22:49.790 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: Command line: ntpd -gq 2026-02-22T22:22:49.790 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: ---------------------------------------------------- 2026-02-22T22:22:49.790 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: ntp-4 is maintained by Network Time Foundation, 2026-02-22T22:22:49.790 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2026-02-22T22:22:49.790 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: corporation. Support and training for ntp-4 are 2026-02-22T22:22:49.791 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: available at https://www.nwtime.org/support 2026-02-22T22:22:49.791 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: ---------------------------------------------------- 2026-02-22T22:22:49.791 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: proto: precision = 0.030 usec (-25) 2026-02-22T22:22:49.791 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: basedate set to 2022-02-04 2026-02-22T22:22:49.791 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: gps base set to 2022-02-06 (week 2196) 2026-02-22T22:22:49.791 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-22T22:22:49.791 INFO:teuthology.orchestra.run.trial190.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2026-02-22T22:22:49.791 INFO:teuthology.orchestra.run.trial190.stderr:restrict ::: KOD does nothing without LIMITED. 2026-02-22T22:22:49.793 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: restrict ::: KOD does nothing without LIMITED. 2026-02-22T22:22:49.793 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: Listen and drop on 0 v6wildcard [::]:123 2026-02-22T22:22:49.793 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2026-02-22T22:22:49.793 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: Listen normally on 2 lo 127.0.0.1:123 2026-02-22T22:22:49.793 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: Listen normally on 3 enp1s0f0 10.20.193.190:123 2026-02-22T22:22:49.793 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: Listen normally on 4 lo [::1]:123 2026-02-22T22:22:49.793 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: Listen normally on 5 enp1s0f0 [fe80::925a:8ff:fe77:5f90%2]:123 2026-02-22T22:22:49.793 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:50 ntpd[9413]: Listening on routing socket on fd #22 for interface updates 2026-02-22T22:22:50.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:50 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:50.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:50 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:50.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:51 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:50.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:51 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:50.839 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:50 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:50.840 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:50 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:50.840 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:50 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:50.840 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:50 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:50.855 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:51 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:50.855 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:51 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:50.855 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:51 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:50.855 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:51 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:51.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:51 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:51.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:51 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:51.789 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:52 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:51.789 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:52 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:51.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:51 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:51.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:51 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:51.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:51 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:51.834 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:51 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:51.859 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:52 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:51.859 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:52 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:51.859 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:52 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:51.859 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:52 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:52.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:52.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:52.836 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.836 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.836 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:52.836 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:52.849 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.849 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.849 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:52.849 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:52 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:52.854 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.855 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.855 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:52.855 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:52.856 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.857 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.857 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:52.857 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:52.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:52.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:52.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:52.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:53 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:53.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:53 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:53.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:53 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:53.789 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:54 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:53.789 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:54 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:53.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:53 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:53.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:53 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:53.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:53 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:53.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:53 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:53.863 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:54 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:53.863 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:54 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:53.863 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:54 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:53.863 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:54 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:54.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.832 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.832 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:54.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:54.840 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.840 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.840 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:54.840 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:54.848 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.848 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.849 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:54.849 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:54 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:54.854 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.854 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.855 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:54.855 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:54.856 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.856 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.856 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:54.857 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:54.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:54.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:54.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:54.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:55 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:55.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:55 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:55.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:55 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:55.789 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:56 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:55.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:56 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:55.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:55 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:55.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:55 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:55.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:55 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:55.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:55 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:56.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:56.766 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:56.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:56.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:56.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:56.767 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:56.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:56.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:56.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:56.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:56.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:56.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:56.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:56.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:56.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:56.833 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:56.839 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:56.839 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:56.839 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:56.839 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:56.840 INFO:teuthology.orchestra.run.trial103.stdout:22 Feb 22:22:56 ntpd[9421]: ntpd: time set -0.042451 s 2026-02-22T22:22:56.840 INFO:teuthology.orchestra.run.trial103.stdout:ntpd: time set -0.042451s 2026-02-22T22:22:56.840 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2026-02-22T22:22:56.841 INFO:teuthology.orchestra.run.trial103.stderr:22 Feb 22:22:56 ntpd[9421]: can't open /var/log/ntpstats/loopstats.20260222: Permission denied 2026-02-22T22:22:56.856 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:56.856 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:56.857 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:56.857 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:56.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:56.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:56.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:56.872 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:57 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:56.890 INFO:teuthology.orchestra.run.trial103.stdout: remote refid st t when poll reach delay offset jitter 2026-02-22T22:22:56.890 INFO:teuthology.orchestra.run.trial103.stdout:============================================================================== 2026-02-22T22:22:56.890 INFO:teuthology.orchestra.run.trial103.stdout: clock.nyc.he.ne .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T22:22:56.890 INFO:teuthology.orchestra.run.trial103.stdout: 170-187-147-56. .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T22:22:56.890 INFO:teuthology.orchestra.run.trial103.stdout: 172-234-44-141. .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T22:22:56.890 INFO:teuthology.orchestra.run.trial103.stdout: 149.28.61.105 ( .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T22:22:57.789 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:58 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:57.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:58 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:57.862 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:58 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:57.863 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:58 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:57.863 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:58 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:57.863 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:58 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:58.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:59 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:58.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:59 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:58.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:59 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:58.790 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:59 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:58.857 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:59 ntpd[9413]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2026-02-22T22:22:58.857 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:59 ntpd[9413]: can't open /var/log/ntpstats/rawstats.20260222: Permission denied 2026-02-22T22:22:58.857 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:59 ntpd[9413]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2026-02-22T22:22:58.857 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:59 ntpd[9413]: can't open /var/log/ntpstats/peerstats.20260222: Permission denied 2026-02-22T22:22:58.857 INFO:teuthology.orchestra.run.trial190.stdout:22 Feb 22:22:58 ntpd[9413]: ntpd: time set -0.283636 s 2026-02-22T22:22:58.857 INFO:teuthology.orchestra.run.trial190.stdout:ntpd: time set -0.283636s 2026-02-22T22:22:58.858 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:58 ntpd[9413]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2026-02-22T22:22:58.858 INFO:teuthology.orchestra.run.trial190.stderr:22 Feb 22:22:58 ntpd[9413]: can't open /var/log/ntpstats/loopstats.20260222: Permission denied 2026-02-22T22:22:58.938 INFO:teuthology.orchestra.run.trial190.stdout: remote refid st t when poll reach delay offset jitter 2026-02-22T22:22:58.938 INFO:teuthology.orchestra.run.trial190.stdout:============================================================================== 2026-02-22T22:22:58.938 INFO:teuthology.orchestra.run.trial190.stdout: 149.248.12.167 .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T22:22:58.938 INFO:teuthology.orchestra.run.trial190.stdout: europa.ellipse. .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T22:22:58.938 INFO:teuthology.orchestra.run.trial190.stdout: ntp.maxhost.io .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T22:22:58.939 INFO:teuthology.orchestra.run.trial190.stdout: 172-233-177-198 .INIT. 16 u - 64 0 0.000 +0.000 0.000 2026-02-22T22:22:58.940 INFO:teuthology.run_tasks:Running task install... 2026-02-22T22:22:58.955 DEBUG:teuthology.task.install:project ceph 2026-02-22T22:22:58.955 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}} 2026-02-22T22:22:58.955 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'} 2026-02-22T22:22:58.955 INFO:teuthology.task.install:Using flavor: default 2026-02-22T22:22:58.964 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-22T22:22:58.964 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-22T22:22:58.965 DEBUG:teuthology.orchestra.run.trial103:> sudo apt-key list | grep Ceph 2026-02-22T22:22:58.966 DEBUG:teuthology.orchestra.run.trial190:> sudo apt-key list | grep Ceph 2026-02-22T22:22:59.012 INFO:teuthology.orchestra.run.trial103.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-02-22T22:22:59.026 INFO:teuthology.orchestra.run.trial190.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2026-02-22T22:22:59.041 INFO:teuthology.orchestra.run.trial103.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-02-22T22:22:59.041 INFO:teuthology.orchestra.run.trial103.stdout:uid [ unknown] Ceph.com (release key) 2026-02-22T22:22:59.041 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-02-22T22:22:59.042 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:22:59.055 INFO:teuthology.orchestra.run.trial190.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2026-02-22T22:22:59.055 INFO:teuthology.orchestra.run.trial190.stdout:uid [ unknown] Ceph.com (release key) 2026-02-22T22:22:59.057 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, ceph-volume, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse, rbd-mirror on remote deb x86_64 2026-02-22T22:22:59.057 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:22:59.213 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default/ 2026-02-22T22:22:59.213 INFO:teuthology.task.install.deb:Package version is 19.2.3-689-gf90f4eae-1jammy 2026-02-22T22:22:59.222 INFO:teuthology.task.install.deb:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default/ 2026-02-22T22:22:59.222 INFO:teuthology.task.install.deb:Package version is 19.2.3-689-gf90f4eae-1jammy 2026-02-22T22:22:59.364 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:22:59.364 DEBUG:teuthology.orchestra.run.trial103:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-02-22T22:22:59.372 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:22:59.372 DEBUG:teuthology.orchestra.run.trial190:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2026-02-22T22:22:59.375 DEBUG:teuthology.orchestra.run.trial103:> sudo apt-get update 2026-02-22T22:22:59.381 DEBUG:teuthology.orchestra.run.trial190:> sudo apt-get update 2026-02-22T22:22:59.531 INFO:teuthology.orchestra.run.trial190.stdout:Ign:1 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy InRelease 2026-02-22T22:22:59.531 INFO:teuthology.orchestra.run.trial103.stdout:Ign:1 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy InRelease 2026-02-22T22:22:59.543 INFO:teuthology.orchestra.run.trial103.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy Release [7662 B] 2026-02-22T22:22:59.543 INFO:teuthology.orchestra.run.trial190.stdout:Get:2 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy Release [7662 B] 2026-02-22T22:22:59.555 INFO:teuthology.orchestra.run.trial103.stdout:Ign:3 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy Release.gpg 2026-02-22T22:22:59.555 INFO:teuthology.orchestra.run.trial190.stdout:Ign:3 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy Release.gpg 2026-02-22T22:22:59.566 INFO:teuthology.orchestra.run.trial103.stdout:Get:4 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.1 kB] 2026-02-22T22:22:59.567 INFO:teuthology.orchestra.run.trial190.stdout:Get:4 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 Packages [18.1 kB] 2026-02-22T22:22:59.889 INFO:teuthology.orchestra.run.trial190.stdout:Hit:5 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-22T22:22:59.905 INFO:teuthology.orchestra.run.trial103.stdout:Hit:5 https://archive.ubuntu.com/ubuntu jammy InRelease 2026-02-22T22:22:59.971 INFO:teuthology.orchestra.run.trial190.stdout:Hit:6 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-22T22:22:59.991 INFO:teuthology.orchestra.run.trial103.stdout:Hit:6 https://archive.ubuntu.com/ubuntu jammy-updates InRelease 2026-02-22T22:23:00.059 INFO:teuthology.orchestra.run.trial190.stdout:Hit:7 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-22T22:23:00.082 INFO:teuthology.orchestra.run.trial103.stdout:Hit:7 https://archive.ubuntu.com/ubuntu jammy-security InRelease 2026-02-22T22:23:00.146 INFO:teuthology.orchestra.run.trial190.stdout:Hit:8 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-22T22:23:00.174 INFO:teuthology.orchestra.run.trial103.stdout:Hit:8 https://archive.ubuntu.com/ubuntu jammy-backports InRelease 2026-02-22T22:23:00.288 INFO:teuthology.orchestra.run.trial190.stdout:Fetched 25.8 kB in 1s (34.6 kB/s) 2026-02-22T22:23:00.317 INFO:teuthology.orchestra.run.trial103.stdout:Fetched 25.8 kB in 1s (33.0 kB/s) 2026-02-22T22:23:00.804 INFO:teuthology.orchestra.run.trial190.stdout:Reading package lists... 2026-02-22T22:23:00.817 DEBUG:teuthology.orchestra.run.trial190:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.2.3-689-gf90f4eae-1jammy cephadm=19.2.3-689-gf90f4eae-1jammy ceph-mds=19.2.3-689-gf90f4eae-1jammy ceph-mgr=19.2.3-689-gf90f4eae-1jammy ceph-common=19.2.3-689-gf90f4eae-1jammy ceph-fuse=19.2.3-689-gf90f4eae-1jammy ceph-test=19.2.3-689-gf90f4eae-1jammy ceph-volume=19.2.3-689-gf90f4eae-1jammy radosgw=19.2.3-689-gf90f4eae-1jammy python3-rados=19.2.3-689-gf90f4eae-1jammy python3-rgw=19.2.3-689-gf90f4eae-1jammy python3-cephfs=19.2.3-689-gf90f4eae-1jammy python3-rbd=19.2.3-689-gf90f4eae-1jammy libcephfs2=19.2.3-689-gf90f4eae-1jammy libcephfs-dev=19.2.3-689-gf90f4eae-1jammy librados2=19.2.3-689-gf90f4eae-1jammy librbd1=19.2.3-689-gf90f4eae-1jammy rbd-fuse=19.2.3-689-gf90f4eae-1jammy rbd-mirror=19.2.3-689-gf90f4eae-1jammy 2026-02-22T22:23:00.842 INFO:teuthology.orchestra.run.trial103.stdout:Reading package lists... 2026-02-22T22:23:00.849 INFO:teuthology.orchestra.run.trial190.stdout:Reading package lists... 2026-02-22T22:23:00.855 DEBUG:teuthology.orchestra.run.trial103:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=19.2.3-689-gf90f4eae-1jammy cephadm=19.2.3-689-gf90f4eae-1jammy ceph-mds=19.2.3-689-gf90f4eae-1jammy ceph-mgr=19.2.3-689-gf90f4eae-1jammy ceph-common=19.2.3-689-gf90f4eae-1jammy ceph-fuse=19.2.3-689-gf90f4eae-1jammy ceph-test=19.2.3-689-gf90f4eae-1jammy ceph-volume=19.2.3-689-gf90f4eae-1jammy radosgw=19.2.3-689-gf90f4eae-1jammy python3-rados=19.2.3-689-gf90f4eae-1jammy python3-rgw=19.2.3-689-gf90f4eae-1jammy python3-cephfs=19.2.3-689-gf90f4eae-1jammy python3-rbd=19.2.3-689-gf90f4eae-1jammy libcephfs2=19.2.3-689-gf90f4eae-1jammy libcephfs-dev=19.2.3-689-gf90f4eae-1jammy librados2=19.2.3-689-gf90f4eae-1jammy librbd1=19.2.3-689-gf90f4eae-1jammy rbd-fuse=19.2.3-689-gf90f4eae-1jammy rbd-mirror=19.2.3-689-gf90f4eae-1jammy 2026-02-22T22:23:00.888 INFO:teuthology.orchestra.run.trial103.stdout:Reading package lists... 2026-02-22T22:23:00.958 INFO:teuthology.orchestra.run.trial190.stdout:Building dependency tree... 2026-02-22T22:23:00.958 INFO:teuthology.orchestra.run.trial190.stdout:Reading state information... 2026-02-22T22:23:00.994 INFO:teuthology.orchestra.run.trial103.stdout:Building dependency tree... 2026-02-22T22:23:00.995 INFO:teuthology.orchestra.run.trial103.stdout:Reading state information... 2026-02-22T22:23:01.048 INFO:teuthology.orchestra.run.trial190.stdout:The following packages were automatically installed and are no longer required: 2026-02-22T22:23:01.048 INFO:teuthology.orchestra.run.trial190.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-22T22:23:01.048 INFO:teuthology.orchestra.run.trial190.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-22T22:23:01.048 INFO:teuthology.orchestra.run.trial190.stdout: sg3-utils-udev 2026-02-22T22:23:01.048 INFO:teuthology.orchestra.run.trial190.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-22T22:23:01.049 INFO:teuthology.orchestra.run.trial190.stdout:The following additional packages will be installed: 2026-02-22T22:23:01.049 INFO:teuthology.orchestra.run.trial190.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-02-22T22:23:01.049 INFO:teuthology.orchestra.run.trial190.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2026-02-22T22:23:01.049 INFO:teuthology.orchestra.run.trial190.stdout: libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 liblua5.3-dev libnbd0 2026-02-22T22:23:01.049 INFO:teuthology.orchestra.run.trial190.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2026-02-22T22:23:01.049 INFO:teuthology.orchestra.run.trial190.stdout: libradosstriper1 librdkafka1 libreadline-dev librgw2 libsqlite3-mod-ceph 2026-02-22T22:23:01.049 INFO:teuthology.orchestra.run.trial190.stdout: libthrift-0.16.0 lua-any lua-sec lua-socket lua5.1 luarocks nvme-cli 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: pkg-config python-asyncssh-doc python-pastedeploy-tpl python3-asyncssh 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cheroot 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-logutils 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-mako python3-natsort python3-paste python3-pastedeploy 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-pyinotify python3-pytest python3-repoze.lru 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplegeneric 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-simplejson python3-singledispatch python3-sklearn 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-sklearn-lib python3-tempita python3-tempora python3-threadpoolctl 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-toml python3-waitress python3-wcwidth python3-webob 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-websocket python3-webtest python3-werkzeug python3-zc.lockfile 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: qttranslations5-l10n socat unzip xmlstarlet zip 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout:Suggested packages: 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python3-influxdb readline-doc python3-beaker python-mako-doc 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python-natsort-doc httpd-wsgi libapache2-mod-python libapache2-mod-scgi 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: libjs-mochikit python-pecan-doc python-psutil-doc subversion 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python-pygments-doc ttf-bitstream-vera python-pyinotify-doc python3-dap 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: python-webtest-doc python-werkzeug-doc python3-watchdog 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout:Recommended packages: 2026-02-22T22:23:01.050 INFO:teuthology.orchestra.run.trial190.stdout: btrfs-tools 2026-02-22T22:23:01.081 INFO:teuthology.orchestra.run.trial103.stdout:The following packages were automatically installed and are no longer required: 2026-02-22T22:23:01.081 INFO:teuthology.orchestra.run.trial103.stdout: kpartx libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 2026-02-22T22:23:01.081 INFO:teuthology.orchestra.run.trial103.stdout: libgfxdr0 libglusterfs0 libiscsi7 libpmemobj1 libsgutils2-2 sg3-utils 2026-02-22T22:23:01.081 INFO:teuthology.orchestra.run.trial103.stdout: sg3-utils-udev 2026-02-22T22:23:01.081 INFO:teuthology.orchestra.run.trial103.stdout:Use 'sudo apt autoremove' to remove them. 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout:The following additional packages will be installed: 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd jq 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 liblua5.3-dev libnbd0 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: libradosstriper1 librdkafka1 libreadline-dev librgw2 libsqlite3-mod-ceph 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: libthrift-0.16.0 lua-any lua-sec lua-socket lua5.1 luarocks nvme-cli 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: pkg-config python-asyncssh-doc python-pastedeploy-tpl python3-asyncssh 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cheroot 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-logutils 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: python3-mako python3-natsort python3-paste python3-pastedeploy 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2026-02-22T22:23:01.082 INFO:teuthology.orchestra.run.trial103.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python3-pyinotify python3-pytest python3-repoze.lru 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python3-requests-oauthlib python3-routes python3-rsa python3-simplegeneric 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python3-simplejson python3-singledispatch python3-sklearn 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python3-sklearn-lib python3-tempita python3-tempora python3-threadpoolctl 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python3-toml python3-waitress python3-wcwidth python3-webob 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python3-websocket python3-webtest python3-werkzeug python3-zc.lockfile 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: qttranslations5-l10n socat unzip xmlstarlet zip 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout:Suggested packages: 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python3-influxdb readline-doc python3-beaker python-mako-doc 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python-natsort-doc httpd-wsgi libapache2-mod-python libapache2-mod-scgi 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: libjs-mochikit python-pecan-doc python-psutil-doc subversion 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python-pygments-doc ttf-bitstream-vera python-pyinotify-doc python3-dap 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python-sklearn-doc ipython3 python-waitress-doc python-webob-doc 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: python-webtest-doc python-werkzeug-doc python3-watchdog 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout:Recommended packages: 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial103.stdout: btrfs-tools 2026-02-22T22:23:01.083 INFO:teuthology.orchestra.run.trial190.stdout:The following NEW packages will be installed: 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: liblua5.3-dev libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: libqt5dbus5 libqt5network5 librados2 libradosstriper1 librbd1 librdkafka1 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 lua-any lua-sec 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: lua-socket lua5.1 luarocks nvme-cli pkg-config python-asyncssh-doc 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: python-pastedeploy-tpl python3-asyncssh python3-cachetools 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: python3-ceph-argparse python3-ceph-common python3-cephfs python3-cheroot 2026-02-22T22:23:01.084 INFO:teuthology.orchestra.run.trial190.stdout: python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-logutils 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-mako python3-natsort python3-paste python3-pastedeploy 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-pyinotify python3-pytest python3-rados python3-rbd 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-repoze.lru python3-requests-oauthlib python3-rgw python3-routes 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-threadpoolctl python3-toml python3-waitress python3-wcwidth 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-webob python3-websocket python3-webtest python3-werkzeug 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse rbd-mirror socat 2026-02-22T22:23:01.085 INFO:teuthology.orchestra.run.trial190.stdout: unzip xmlstarlet zip 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout:The following NEW packages will be installed: 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfuse2 libjq1 liblttng-ust1 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: liblua5.3-dev libnbd0 liboath0 libonig5 libpcre2-16-0 libqt5core5a 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: libqt5dbus5 libqt5network5 librados2 libradosstriper1 librbd1 librdkafka1 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: libreadline-dev librgw2 libsqlite3-mod-ceph libthrift-0.16.0 lua-any lua-sec 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: lua-socket lua5.1 luarocks nvme-cli pkg-config python-asyncssh-doc 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: python-pastedeploy-tpl python3-asyncssh python3-cachetools 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: python3-ceph-argparse python3-ceph-common python3-cephfs python3-cheroot 2026-02-22T22:23:01.117 INFO:teuthology.orchestra.run.trial103.stdout: python3-cherrypy3 python3-google-auth python3-iniconfig 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-jaraco.text python3-joblib python3-kubernetes python3-logutils 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-mako python3-natsort python3-paste python3-pastedeploy 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-prettytable python3-psutil python3-py python3-pygments 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-pyinotify python3-pytest python3-rados python3-rbd 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-repoze.lru python3-requests-oauthlib python3-rgw python3-routes 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-threadpoolctl python3-toml python3-waitress python3-wcwidth 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-webob python3-websocket python3-webtest python3-werkzeug 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse rbd-mirror socat 2026-02-22T22:23:01.118 INFO:teuthology.orchestra.run.trial103.stdout: unzip xmlstarlet zip 2026-02-22T22:23:01.174 INFO:teuthology.orchestra.run.trial190.stdout:0 upgraded, 109 newly installed, 0 to remove and 13 not upgraded. 2026-02-22T22:23:01.174 INFO:teuthology.orchestra.run.trial190.stdout:Need to get 181 MB of archives. 2026-02-22T22:23:01.174 INFO:teuthology.orchestra.run.trial190.stdout:After this operation, 821 MB of additional disk space will be used. 2026-02-22T22:23:01.174 INFO:teuthology.orchestra.run.trial190.stdout:Get:1 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 19.2.3-689-gf90f4eae-1jammy [3597 kB] 2026-02-22T22:23:01.189 INFO:teuthology.orchestra.run.trial190.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-02-22T22:23:01.221 INFO:teuthology.orchestra.run.trial103.stdout:0 upgraded, 109 newly installed, 0 to remove and 13 not upgraded. 2026-02-22T22:23:01.221 INFO:teuthology.orchestra.run.trial103.stdout:Need to get 181 MB of archives. 2026-02-22T22:23:01.221 INFO:teuthology.orchestra.run.trial103.stdout:After this operation, 821 MB of additional disk space will be used. 2026-02-22T22:23:01.221 INFO:teuthology.orchestra.run.trial103.stdout:Get:1 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 19.2.3-689-gf90f4eae-1jammy [3597 kB] 2026-02-22T22:23:01.235 INFO:teuthology.orchestra.run.trial190.stdout:Get:3 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-02-22T22:23:01.236 INFO:teuthology.orchestra.run.trial190.stdout:Get:4 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-02-22T22:23:01.245 INFO:teuthology.orchestra.run.trial190.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-02-22T22:23:01.273 INFO:teuthology.orchestra.run.trial190.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-02-22T22:23:01.274 INFO:teuthology.orchestra.run.trial190.stdout:Get:7 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-02-22T22:23:01.279 INFO:teuthology.orchestra.run.trial190.stdout:Get:8 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-02-22T22:23:01.281 INFO:teuthology.orchestra.run.trial190.stdout:Get:9 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-02-22T22:23:01.282 INFO:teuthology.orchestra.run.trial190.stdout:Get:10 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-02-22T22:23:01.283 INFO:teuthology.orchestra.run.trial190.stdout:Get:11 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-02-22T22:23:01.283 INFO:teuthology.orchestra.run.trial190.stdout:Get:12 https://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-02-22T22:23:01.287 INFO:teuthology.orchestra.run.trial190.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy/main amd64 libreadline-dev amd64 8.1.2-1 [166 kB] 2026-02-22T22:23:01.289 INFO:teuthology.orchestra.run.trial190.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblua5.3-dev amd64 5.3.6-1build1 [167 kB] 2026-02-22T22:23:01.290 INFO:teuthology.orchestra.run.trial190.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua5.1 amd64 5.1.5-8.1build4 [94.6 kB] 2026-02-22T22:23:01.294 INFO:teuthology.orchestra.run.trial190.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-any all 27ubuntu1 [5034 B] 2026-02-22T22:23:01.294 INFO:teuthology.orchestra.run.trial190.stdout:Get:17 https://archive.ubuntu.com/ubuntu jammy/main amd64 zip amd64 3.0-12build2 [176 kB] 2026-02-22T22:23:01.295 INFO:teuthology.orchestra.run.trial190.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.2 [175 kB] 2026-02-22T22:23:01.296 INFO:teuthology.orchestra.run.trial190.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy/universe amd64 luarocks all 3.8.0+dfsg1-1 [140 kB] 2026-02-22T22:23:01.298 INFO:teuthology.orchestra.run.trial190.stdout:Get:20 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-02-22T22:23:01.303 INFO:teuthology.orchestra.run.trial190.stdout:Get:21 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-02-22T22:23:01.303 INFO:teuthology.orchestra.run.trial190.stdout:Get:22 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-02-22T22:23:01.304 INFO:teuthology.orchestra.run.trial190.stdout:Get:23 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-02-22T22:23:01.305 INFO:teuthology.orchestra.run.trial190.stdout:Get:24 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-02-22T22:23:01.305 INFO:teuthology.orchestra.run.trial190.stdout:Get:25 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-02-22T22:23:01.313 INFO:teuthology.orchestra.run.trial190.stdout:Get:26 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-02-22T22:23:01.313 INFO:teuthology.orchestra.run.trial190.stdout:Get:27 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-02-22T22:23:01.314 INFO:teuthology.orchestra.run.trial190.stdout:Get:28 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-02-22T22:23:01.314 INFO:teuthology.orchestra.run.trial190.stdout:Get:29 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-02-22T22:23:01.315 INFO:teuthology.orchestra.run.trial190.stdout:Get:30 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-02-22T22:23:01.322 INFO:teuthology.orchestra.run.trial190.stdout:Get:31 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2026-02-22T22:23:01.323 INFO:teuthology.orchestra.run.trial190.stdout:Get:32 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2026-02-22T22:23:01.323 INFO:teuthology.orchestra.run.trial190.stdout:Get:33 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2026-02-22T22:23:01.324 INFO:teuthology.orchestra.run.trial190.stdout:Get:34 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7320 B] 2026-02-22T22:23:01.324 INFO:teuthology.orchestra.run.trial190.stdout:Get:35 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-02-22T22:23:01.332 INFO:teuthology.orchestra.run.trial190.stdout:Get:36 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-waitress all 1.4.4-1.1ubuntu1.1 [47.0 kB] 2026-02-22T22:23:01.332 INFO:teuthology.orchestra.run.trial190.stdout:Get:37 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2026-02-22T22:23:01.333 INFO:teuthology.orchestra.run.trial190.stdout:Get:38 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2026-02-22T22:23:01.335 INFO:teuthology.orchestra.run.trial190.stdout:Get:39 https://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4892 B] 2026-02-22T22:23:01.336 INFO:teuthology.orchestra.run.trial190.stdout:Get:40 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2026-02-22T22:23:01.341 INFO:teuthology.orchestra.run.trial190.stdout:Get:41 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2026-02-22T22:23:01.341 INFO:teuthology.orchestra.run.trial190.stdout:Get:42 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2026-02-22T22:23:01.342 INFO:teuthology.orchestra.run.trial190.stdout:Get:43 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.3 [181 kB] 2026-02-22T22:23:01.344 INFO:teuthology.orchestra.run.trial190.stdout:Get:44 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-02-22T22:23:01.344 INFO:teuthology.orchestra.run.trial190.stdout:Get:45 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-02-22T22:23:01.351 INFO:teuthology.orchestra.run.trial190.stdout:Get:46 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-02-22T22:23:01.351 INFO:teuthology.orchestra.run.trial190.stdout:Get:47 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-02-22T22:23:01.352 INFO:teuthology.orchestra.run.trial190.stdout:Get:48 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-02-22T22:23:01.365 INFO:teuthology.orchestra.run.trial190.stdout:Get:49 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-02-22T22:23:01.367 INFO:teuthology.orchestra.run.trial190.stdout:Get:50 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-02-22T22:23:01.367 INFO:teuthology.orchestra.run.trial190.stdout:Get:51 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-02-22T22:23:01.377 INFO:teuthology.orchestra.run.trial190.stdout:Get:52 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-02-22T22:23:01.377 INFO:teuthology.orchestra.run.trial190.stdout:Get:53 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-02-22T22:23:01.377 INFO:teuthology.orchestra.run.trial190.stdout:Get:54 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-02-22T22:23:01.377 INFO:teuthology.orchestra.run.trial190.stdout:Get:55 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-02-22T22:23:01.378 INFO:teuthology.orchestra.run.trial190.stdout:Get:56 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-02-22T22:23:01.378 INFO:teuthology.orchestra.run.trial190.stdout:Get:57 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-02-22T22:23:01.379 INFO:teuthology.orchestra.run.trial190.stdout:Get:58 https://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-02-22T22:23:01.380 INFO:teuthology.orchestra.run.trial190.stdout:Get:59 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-02-22T22:23:01.381 INFO:teuthology.orchestra.run.trial190.stdout:Get:60 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-02-22T22:23:01.389 INFO:teuthology.orchestra.run.trial190.stdout:Get:61 https://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-02-22T22:23:01.391 INFO:teuthology.orchestra.run.trial190.stdout:Get:62 https://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-02-22T22:23:01.393 INFO:teuthology.orchestra.run.trial190.stdout:Get:63 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-socket amd64 3.0~rc1+git+ac3201d-6 [78.9 kB] 2026-02-22T22:23:01.393 INFO:teuthology.orchestra.run.trial190.stdout:Get:64 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-sec amd64 1.0.2-1 [37.6 kB] 2026-02-22T22:23:01.394 INFO:teuthology.orchestra.run.trial190.stdout:Get:65 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-02-22T22:23:01.396 INFO:teuthology.orchestra.run.trial190.stdout:Get:66 https://archive.ubuntu.com/ubuntu jammy/main amd64 pkg-config amd64 0.29.2-1ubuntu3 [48.2 kB] 2026-02-22T22:23:01.398 INFO:teuthology.orchestra.run.trial190.stdout:Get:67 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-02-22T22:23:01.400 INFO:teuthology.orchestra.run.trial190.stdout:Get:68 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-02-22T22:23:01.400 INFO:teuthology.orchestra.run.trial190.stdout:Get:69 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2026-02-22T22:23:01.400 INFO:teuthology.orchestra.run.trial190.stdout:Get:70 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-02-22T22:23:01.408 INFO:teuthology.orchestra.run.trial190.stdout:Get:71 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-02-22T22:23:01.409 INFO:teuthology.orchestra.run.trial190.stdout:Get:72 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-02-22T22:23:01.410 INFO:teuthology.orchestra.run.trial190.stdout:Get:73 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-02-22T22:23:01.415 INFO:teuthology.orchestra.run.trial190.stdout:Get:74 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2026-02-22T22:23:01.415 INFO:teuthology.orchestra.run.trial190.stdout:Get:75 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-02-22T22:23:01.415 INFO:teuthology.orchestra.run.trial190.stdout:Get:76 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-02-22T22:23:01.417 INFO:teuthology.orchestra.run.trial190.stdout:Get:77 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-02-22T22:23:01.418 INFO:teuthology.orchestra.run.trial190.stdout:Get:78 https://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-02-22T22:23:01.471 INFO:teuthology.orchestra.run.trial103.stdout:Get:2 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2026-02-22T22:23:01.659 INFO:teuthology.orchestra.run.trial190.stdout:Get:79 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 19.2.3-689-gf90f4eae-1jammy [3256 kB] 2026-02-22T22:23:01.696 INFO:teuthology.orchestra.run.trial103.stdout:Get:3 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 19.2.3-689-gf90f4eae-1jammy [3256 kB] 2026-02-22T22:23:01.761 INFO:teuthology.orchestra.run.trial190.stdout:Get:80 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 19.2.3-689-gf90f4eae-1jammy [979 kB] 2026-02-22T22:23:01.792 INFO:teuthology.orchestra.run.trial190.stdout:Get:81 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 19.2.3-689-gf90f4eae-1jammy [357 kB] 2026-02-22T22:23:01.795 INFO:teuthology.orchestra.run.trial190.stdout:Get:82 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 19.2.3-689-gf90f4eae-1jammy [32.9 kB] 2026-02-22T22:23:01.796 INFO:teuthology.orchestra.run.trial190.stdout:Get:83 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 19.2.3-689-gf90f4eae-1jammy [184 kB] 2026-02-22T22:23:01.805 INFO:teuthology.orchestra.run.trial103.stdout:Get:4 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 19.2.3-689-gf90f4eae-1jammy [979 kB] 2026-02-22T22:23:01.806 INFO:teuthology.orchestra.run.trial190.stdout:Get:84 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 19.2.3-689-gf90f4eae-1jammy [70.0 kB] 2026-02-22T22:23:01.807 INFO:teuthology.orchestra.run.trial190.stdout:Get:85 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 19.2.3-689-gf90f4eae-1jammy [335 kB] 2026-02-22T22:23:01.808 INFO:teuthology.orchestra.run.trial103.stdout:Get:5 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2026-02-22T22:23:01.812 INFO:teuthology.orchestra.run.trial190.stdout:Get:86 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 19.2.3-689-gf90f4eae-1jammy [6935 kB] 2026-02-22T22:23:01.819 INFO:teuthology.orchestra.run.trial103.stdout:Get:6 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2026-02-22T22:23:01.846 INFO:teuthology.orchestra.run.trial103.stdout:Get:7 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 19.2.3-689-gf90f4eae-1jammy [357 kB] 2026-02-22T22:23:01.860 INFO:teuthology.orchestra.run.trial103.stdout:Get:8 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 19.2.3-689-gf90f4eae-1jammy [32.9 kB] 2026-02-22T22:23:01.862 INFO:teuthology.orchestra.run.trial103.stdout:Get:9 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 19.2.3-689-gf90f4eae-1jammy [184 kB] 2026-02-22T22:23:01.865 INFO:teuthology.orchestra.run.trial103.stdout:Get:10 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 19.2.3-689-gf90f4eae-1jammy [70.0 kB] 2026-02-22T22:23:01.866 INFO:teuthology.orchestra.run.trial103.stdout:Get:11 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 19.2.3-689-gf90f4eae-1jammy [335 kB] 2026-02-22T22:23:01.881 INFO:teuthology.orchestra.run.trial103.stdout:Get:12 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 19.2.3-689-gf90f4eae-1jammy [6935 kB] 2026-02-22T22:23:01.889 INFO:teuthology.orchestra.run.trial103.stdout:Get:13 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2006 kB] 2026-02-22T22:23:02.024 INFO:teuthology.orchestra.run.trial190.stdout:Get:87 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 19.2.3-689-gf90f4eae-1jammy [112 kB] 2026-02-22T22:23:02.024 INFO:teuthology.orchestra.run.trial190.stdout:Get:88 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 19.2.3-689-gf90f4eae-1jammy [470 kB] 2026-02-22T22:23:02.044 INFO:teuthology.orchestra.run.trial190.stdout:Get:89 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 19.2.3-689-gf90f4eae-1jammy [26.5 MB] 2026-02-22T22:23:02.089 INFO:teuthology.orchestra.run.trial103.stdout:Get:14 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2026-02-22T22:23:02.100 INFO:teuthology.orchestra.run.trial103.stdout:Get:15 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2026-02-22T22:23:02.126 INFO:teuthology.orchestra.run.trial103.stdout:Get:16 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2026-02-22T22:23:02.126 INFO:teuthology.orchestra.run.trial103.stdout:Get:17 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 19.2.3-689-gf90f4eae-1jammy [112 kB] 2026-02-22T22:23:02.135 INFO:teuthology.orchestra.run.trial103.stdout:Get:18 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libnbd0 amd64 1.10.5-1 [71.3 kB] 2026-02-22T22:23:02.137 INFO:teuthology.orchestra.run.trial103.stdout:Get:19 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2026-02-22T22:23:02.137 INFO:teuthology.orchestra.run.trial103.stdout:Get:20 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2026-02-22T22:23:02.137 INFO:teuthology.orchestra.run.trial103.stdout:Get:21 https://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2026-02-22T22:23:02.143 INFO:teuthology.orchestra.run.trial103.stdout:Get:22 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 19.2.3-689-gf90f4eae-1jammy [470 kB] 2026-02-22T22:23:02.152 INFO:teuthology.orchestra.run.trial103.stdout:Get:23 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 19.2.3-689-gf90f4eae-1jammy [26.5 MB] 2026-02-22T22:23:02.155 INFO:teuthology.orchestra.run.trial103.stdout:Get:24 https://archive.ubuntu.com/ubuntu jammy/main amd64 libreadline-dev amd64 8.1.2-1 [166 kB] 2026-02-22T22:23:02.159 INFO:teuthology.orchestra.run.trial103.stdout:Get:25 https://archive.ubuntu.com/ubuntu jammy/main amd64 liblua5.3-dev amd64 5.3.6-1build1 [167 kB] 2026-02-22T22:23:02.162 INFO:teuthology.orchestra.run.trial103.stdout:Get:26 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua5.1 amd64 5.1.5-8.1build4 [94.6 kB] 2026-02-22T22:23:02.231 INFO:teuthology.orchestra.run.trial103.stdout:Get:27 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-any all 27ubuntu1 [5034 B] 2026-02-22T22:23:02.231 INFO:teuthology.orchestra.run.trial103.stdout:Get:28 https://archive.ubuntu.com/ubuntu jammy/main amd64 zip amd64 3.0-12build2 [176 kB] 2026-02-22T22:23:02.233 INFO:teuthology.orchestra.run.trial103.stdout:Get:29 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 unzip amd64 6.0-26ubuntu3.2 [175 kB] 2026-02-22T22:23:02.234 INFO:teuthology.orchestra.run.trial103.stdout:Get:30 https://archive.ubuntu.com/ubuntu jammy/universe amd64 luarocks all 3.8.0+dfsg1-1 [140 kB] 2026-02-22T22:23:02.236 INFO:teuthology.orchestra.run.trial103.stdout:Get:31 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 liboath0 amd64 2.6.7-3ubuntu0.1 [41.3 kB] 2026-02-22T22:23:02.236 INFO:teuthology.orchestra.run.trial103.stdout:Get:32 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9030 B] 2026-02-22T22:23:02.237 INFO:teuthology.orchestra.run.trial103.stdout:Get:33 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2026-02-22T22:23:02.237 INFO:teuthology.orchestra.run.trial103.stdout:Get:34 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6452 B] 2026-02-22T22:23:02.238 INFO:teuthology.orchestra.run.trial103.stdout:Get:35 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8716 B] 2026-02-22T22:23:02.308 INFO:teuthology.orchestra.run.trial103.stdout:Get:36 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2026-02-22T22:23:02.308 INFO:teuthology.orchestra.run.trial103.stdout:Get:37 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2026-02-22T22:23:02.309 INFO:teuthology.orchestra.run.trial103.stdout:Get:38 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7240 B] 2026-02-22T22:23:02.309 INFO:teuthology.orchestra.run.trial103.stdout:Get:39 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8980 B] 2026-02-22T22:23:02.380 INFO:teuthology.orchestra.run.trial103.stdout:Get:40 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2026-02-22T22:23:02.382 INFO:teuthology.orchestra.run.trial103.stdout:Get:41 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2026-02-22T22:23:02.382 INFO:teuthology.orchestra.run.trial103.stdout:Get:42 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2026-02-22T22:23:02.383 INFO:teuthology.orchestra.run.trial103.stdout:Get:43 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2026-02-22T22:23:02.383 INFO:teuthology.orchestra.run.trial103.stdout:Get:44 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2026-02-22T22:23:02.383 INFO:teuthology.orchestra.run.trial103.stdout:Get:45 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7320 B] 2026-02-22T22:23:02.451 INFO:teuthology.orchestra.run.trial103.stdout:Get:46 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2026-02-22T22:23:02.452 INFO:teuthology.orchestra.run.trial103.stdout:Get:47 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-waitress all 1.4.4-1.1ubuntu1.1 [47.0 kB] 2026-02-22T22:23:02.453 INFO:teuthology.orchestra.run.trial103.stdout:Get:48 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2026-02-22T22:23:02.453 INFO:teuthology.orchestra.run.trial103.stdout:Get:49 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2026-02-22T22:23:02.523 INFO:teuthology.orchestra.run.trial103.stdout:Get:50 https://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4892 B] 2026-02-22T22:23:02.523 INFO:teuthology.orchestra.run.trial103.stdout:Get:51 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2026-02-22T22:23:02.524 INFO:teuthology.orchestra.run.trial103.stdout:Get:52 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2026-02-22T22:23:02.524 INFO:teuthology.orchestra.run.trial103.stdout:Get:53 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2026-02-22T22:23:02.525 INFO:teuthology.orchestra.run.trial103.stdout:Get:54 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.3 [181 kB] 2026-02-22T22:23:02.527 INFO:teuthology.orchestra.run.trial103.stdout:Get:55 https://archive.ubuntu.com/ubuntu jammy/universe amd64 libfuse2 amd64 2.9.9-5ubuntu3 [90.3 kB] 2026-02-22T22:23:02.595 INFO:teuthology.orchestra.run.trial103.stdout:Get:56 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python3-asyncssh all 2.5.0-1ubuntu0.1 [189 kB] 2026-02-22T22:23:02.597 INFO:teuthology.orchestra.run.trial103.stdout:Get:57 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2026-02-22T22:23:02.597 INFO:teuthology.orchestra.run.trial103.stdout:Get:58 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2026-02-22T22:23:02.598 INFO:teuthology.orchestra.run.trial103.stdout:Get:59 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2058 kB] 2026-02-22T22:23:02.667 INFO:teuthology.orchestra.run.trial103.stdout:Get:60 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2026-02-22T22:23:02.669 INFO:teuthology.orchestra.run.trial103.stdout:Get:61 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2026-02-22T22:23:02.669 INFO:teuthology.orchestra.run.trial103.stdout:Get:62 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1829 kB] 2026-02-22T22:23:02.689 INFO:teuthology.orchestra.run.trial103.stdout:Get:63 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9722 B] 2026-02-22T22:23:02.690 INFO:teuthology.orchestra.run.trial103.stdout:Get:64 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2026-02-22T22:23:02.690 INFO:teuthology.orchestra.run.trial103.stdout:Get:65 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2026-02-22T22:23:02.739 INFO:teuthology.orchestra.run.trial103.stdout:Get:66 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2026-02-22T22:23:02.739 INFO:teuthology.orchestra.run.trial103.stdout:Get:67 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2026-02-22T22:23:02.739 INFO:teuthology.orchestra.run.trial103.stdout:Get:68 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2026-02-22T22:23:02.743 INFO:teuthology.orchestra.run.trial103.stdout:Get:69 https://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2026-02-22T22:23:02.811 INFO:teuthology.orchestra.run.trial103.stdout:Get:70 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libjq1 amd64 1.6-2.1ubuntu3.1 [133 kB] 2026-02-22T22:23:02.812 INFO:teuthology.orchestra.run.trial103.stdout:Get:71 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 jq amd64 1.6-2.1ubuntu3.1 [52.5 kB] 2026-02-22T22:23:02.812 INFO:teuthology.orchestra.run.trial103.stdout:Get:72 https://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2026-02-22T22:23:02.816 INFO:teuthology.orchestra.run.trial103.stdout:Get:73 https://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2026-02-22T22:23:02.819 INFO:teuthology.orchestra.run.trial103.stdout:Get:74 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-socket amd64 3.0~rc1+git+ac3201d-6 [78.9 kB] 2026-02-22T22:23:02.819 INFO:teuthology.orchestra.run.trial103.stdout:Get:75 https://archive.ubuntu.com/ubuntu jammy/universe amd64 lua-sec amd64 1.0.2-1 [37.6 kB] 2026-02-22T22:23:02.824 INFO:teuthology.orchestra.run.trial190.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 19.2.3-689-gf90f4eae-1jammy [5178 kB] 2026-02-22T22:23:02.882 INFO:teuthology.orchestra.run.trial103.stdout:Get:76 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2026-02-22T22:23:02.887 INFO:teuthology.orchestra.run.trial103.stdout:Get:77 https://archive.ubuntu.com/ubuntu jammy/main amd64 pkg-config amd64 0.29.2-1ubuntu3 [48.2 kB] 2026-02-22T22:23:02.888 INFO:teuthology.orchestra.run.trial103.stdout:Get:78 https://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 python-asyncssh-doc all 2.5.0-1ubuntu0.1 [309 kB] 2026-02-22T22:23:02.891 INFO:teuthology.orchestra.run.trial103.stdout:Get:79 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6024 B] 2026-02-22T22:23:02.955 INFO:teuthology.orchestra.run.trial103.stdout:Get:80 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2026-02-22T22:23:02.955 INFO:teuthology.orchestra.run.trial103.stdout:Get:81 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2026-02-22T22:23:02.956 INFO:teuthology.orchestra.run.trial103.stdout:Get:82 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2026-02-22T22:23:02.957 INFO:teuthology.orchestra.run.trial103.stdout:Get:83 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2026-02-22T22:23:02.957 INFO:teuthology.orchestra.run.trial103.stdout:Get:84 https://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-pygments all 2.11.2+dfsg-2ubuntu0.1 [750 kB] 2026-02-22T22:23:02.965 INFO:teuthology.orchestra.run.trial103.stdout:Get:85 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2026-02-22T22:23:02.996 INFO:teuthology.orchestra.run.trial190.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 19.2.3-689-gf90f4eae-1jammy [248 kB] 2026-02-22T22:23:03.026 INFO:teuthology.orchestra.run.trial103.stdout:Get:86 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2026-02-22T22:23:03.027 INFO:teuthology.orchestra.run.trial103.stdout:Get:87 https://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2026-02-22T22:23:03.029 INFO:teuthology.orchestra.run.trial103.stdout:Get:88 https://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2026-02-22T22:23:03.029 INFO:teuthology.orchestra.run.trial103.stdout:Get:89 https://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1983 kB] 2026-02-22T22:23:03.071 INFO:teuthology.orchestra.run.trial190.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 19.2.3-689-gf90f4eae-1jammy [125 kB] 2026-02-22T22:23:03.072 INFO:teuthology.orchestra.run.trial190.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 19.2.3-689-gf90f4eae-1jammy [1081 kB] 2026-02-22T22:23:03.077 INFO:teuthology.orchestra.run.trial190.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 19.2.3-689-gf90f4eae-1jammy [6239 kB] 2026-02-22T22:23:03.103 INFO:teuthology.orchestra.run.trial103.stdout:Get:90 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 19.2.3-689-gf90f4eae-1jammy [5178 kB] 2026-02-22T22:23:03.222 INFO:teuthology.orchestra.run.trial190.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 19.2.3-689-gf90f4eae-1jammy [23.0 MB] 2026-02-22T22:23:03.291 INFO:teuthology.orchestra.run.trial103.stdout:Get:91 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 19.2.3-689-gf90f4eae-1jammy [248 kB] 2026-02-22T22:23:03.294 INFO:teuthology.orchestra.run.trial103.stdout:Get:92 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 19.2.3-689-gf90f4eae-1jammy [125 kB] 2026-02-22T22:23:03.296 INFO:teuthology.orchestra.run.trial103.stdout:Get:93 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 19.2.3-689-gf90f4eae-1jammy [1081 kB] 2026-02-22T22:23:03.343 INFO:teuthology.orchestra.run.trial103.stdout:Get:94 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 19.2.3-689-gf90f4eae-1jammy [6239 kB] 2026-02-22T22:23:03.564 INFO:teuthology.orchestra.run.trial103.stdout:Get:95 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 19.2.3-689-gf90f4eae-1jammy [23.0 MB] 2026-02-22T22:23:03.923 INFO:teuthology.orchestra.run.trial190.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 19.2.3-689-gf90f4eae-1jammy [14.2 kB] 2026-02-22T22:23:03.923 INFO:teuthology.orchestra.run.trial190.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 19.2.3-689-gf90f4eae-1jammy [1173 kB] 2026-02-22T22:23:03.940 INFO:teuthology.orchestra.run.trial190.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 19.2.3-689-gf90f4eae-1jammy [2503 kB] 2026-02-22T22:23:04.031 INFO:teuthology.orchestra.run.trial190.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 19.2.3-689-gf90f4eae-1jammy [798 kB] 2026-02-22T22:23:04.044 INFO:teuthology.orchestra.run.trial190.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 19.2.3-689-gf90f4eae-1jammy [157 kB] 2026-02-22T22:23:04.046 INFO:teuthology.orchestra.run.trial190.stdout:Get:101 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 19.2.3-689-gf90f4eae-1jammy [2396 kB] 2026-02-22T22:23:04.139 INFO:teuthology.orchestra.run.trial190.stdout:Get:102 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 19.2.3-689-gf90f4eae-1jammy [8625 kB] 2026-02-22T22:23:04.386 INFO:teuthology.orchestra.run.trial190.stdout:Get:103 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 19.2.3-689-gf90f4eae-1jammy [14.3 kB] 2026-02-22T22:23:04.386 INFO:teuthology.orchestra.run.trial190.stdout:Get:104 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 19.2.3-689-gf90f4eae-1jammy [52.1 MB] 2026-02-22T22:23:04.408 INFO:teuthology.orchestra.run.trial103.stdout:Get:96 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 19.2.3-689-gf90f4eae-1jammy [14.2 kB] 2026-02-22T22:23:04.409 INFO:teuthology.orchestra.run.trial103.stdout:Get:97 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 19.2.3-689-gf90f4eae-1jammy [1173 kB] 2026-02-22T22:23:04.424 INFO:teuthology.orchestra.run.trial103.stdout:Get:98 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 19.2.3-689-gf90f4eae-1jammy [2503 kB] 2026-02-22T22:23:04.530 INFO:teuthology.orchestra.run.trial103.stdout:Get:99 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 19.2.3-689-gf90f4eae-1jammy [798 kB] 2026-02-22T22:23:04.543 INFO:teuthology.orchestra.run.trial103.stdout:Get:100 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 19.2.3-689-gf90f4eae-1jammy [157 kB] 2026-02-22T22:23:04.545 INFO:teuthology.orchestra.run.trial103.stdout:Get:101 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 19.2.3-689-gf90f4eae-1jammy [2396 kB] 2026-02-22T22:23:04.650 INFO:teuthology.orchestra.run.trial103.stdout:Get:102 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 19.2.3-689-gf90f4eae-1jammy [8625 kB] 2026-02-22T22:23:04.958 INFO:teuthology.orchestra.run.trial103.stdout:Get:103 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 19.2.3-689-gf90f4eae-1jammy [14.3 kB] 2026-02-22T22:23:04.958 INFO:teuthology.orchestra.run.trial103.stdout:Get:104 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 19.2.3-689-gf90f4eae-1jammy [52.1 MB] 2026-02-22T22:23:05.956 INFO:teuthology.orchestra.run.trial190.stdout:Get:105 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 19.2.3-689-gf90f4eae-1jammy [135 kB] 2026-02-22T22:23:05.956 INFO:teuthology.orchestra.run.trial190.stdout:Get:106 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 19.2.3-689-gf90f4eae-1jammy [41.0 kB] 2026-02-22T22:23:05.957 INFO:teuthology.orchestra.run.trial190.stdout:Get:107 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 19.2.3-689-gf90f4eae-1jammy [13.7 MB] 2026-02-22T22:23:06.364 INFO:teuthology.orchestra.run.trial190.stdout:Get:108 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 19.2.3-689-gf90f4eae-1jammy [92.3 kB] 2026-02-22T22:23:06.365 INFO:teuthology.orchestra.run.trial190.stdout:Get:109 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 19.2.3-689-gf90f4eae-1jammy [3151 kB] 2026-02-22T22:23:06.644 INFO:teuthology.orchestra.run.trial103.stdout:Get:105 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 19.2.3-689-gf90f4eae-1jammy [135 kB] 2026-02-22T22:23:06.644 INFO:teuthology.orchestra.run.trial103.stdout:Get:106 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 19.2.3-689-gf90f4eae-1jammy [41.0 kB] 2026-02-22T22:23:06.645 INFO:teuthology.orchestra.run.trial103.stdout:Get:107 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 19.2.3-689-gf90f4eae-1jammy [13.7 MB] 2026-02-22T22:23:06.792 INFO:teuthology.orchestra.run.trial190.stdout:Fetched 181 MB in 5s (33.6 MB/s) 2026-02-22T22:23:06.814 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-02-22T22:23:06.864 INFO:teuthology.orchestra.run.trial103.stdout:Get:108 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 19.2.3-689-gf90f4eae-1jammy [92.3 kB] 2026-02-22T22:23:06.865 INFO:teuthology.orchestra.run.trial103.stdout:Get:109 https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/ubuntu/jammy/flavors/default jammy/main amd64 rbd-mirror amd64 19.2.3-689-gf90f4eae-1jammy [3151 kB] 2026-02-22T22:23:06.977 INFO:teuthology.orchestra.run.trial190.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 167442 files and directories currently installed.) 2026-02-22T22:23:06.979 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../000-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-02-22T22:23:06.980 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-22T22:23:06.999 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-02-22T22:23:07.012 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../001-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-02-22T22:23:07.012 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-22T22:23:07.028 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-02-22T22:23:07.040 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../002-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-02-22T22:23:07.041 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-22T22:23:07.062 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-02-22T22:23:07.071 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../003-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T22:23:07.076 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:07.110 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-02-22T22:23:07.118 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../004-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T22:23:07.119 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:07.135 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-02-22T22:23:07.143 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../005-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T22:23:07.143 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:07.163 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-02-22T22:23:07.171 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../006-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-02-22T22:23:07.171 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-22T22:23:07.187 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package librados2. 2026-02-22T22:23:07.195 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../007-librados2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.195 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking librados2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.235 INFO:teuthology.orchestra.run.trial103.stdout:Fetched 181 MB in 6s (31.2 MB/s) 2026-02-22T22:23:07.236 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libnbd0. 2026-02-22T22:23:07.244 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../008-libnbd0_1.10.5-1_amd64.deb ... 2026-02-22T22:23:07.244 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-02-22T22:23:07.251 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2026-02-22T22:23:07.258 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package librbd1. 2026-02-22T22:23:07.267 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../009-librbd1_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.267 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking librbd1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.310 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libcephfs2. 2026-02-22T22:23:07.319 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../010-libcephfs2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.319 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libcephfs2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.339 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-rados. 2026-02-22T22:23:07.348 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../011-python3-rados_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.348 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-rados (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.365 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-02-22T22:23:07.373 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../012-python3-ceph-argparse_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:07.374 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-ceph-argparse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.387 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-cephfs. 2026-02-22T22:23:07.395 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../013-python3-cephfs_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.396 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-cephfs (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.408 INFO:teuthology.orchestra.run.trial103.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 167442 files and directories currently installed.) 2026-02-22T22:23:07.410 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../000-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2026-02-22T22:23:07.410 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-ceph-common. 2026-02-22T22:23:07.411 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-22T22:23:07.420 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../014-python3-ceph-common_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:07.420 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.432 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2026-02-22T22:23:07.436 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-wcwidth. 2026-02-22T22:23:07.444 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-02-22T22:23:07.444 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-22T22:23:07.445 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../001-libdouble-conversion3_3.1.7-4_amd64.deb ... 2026-02-22T22:23:07.445 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-22T22:23:07.459 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-prettytable. 2026-02-22T22:23:07.461 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2026-02-22T22:23:07.467 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2026-02-22T22:23:07.468 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-02-22T22:23:07.472 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../002-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2026-02-22T22:23:07.473 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-22T22:23:07.482 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-rbd. 2026-02-22T22:23:07.490 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../017-python3-rbd_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.490 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-rbd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.493 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libqt5core5a:amd64. 2026-02-22T22:23:07.501 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../003-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T22:23:07.506 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:07.508 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-02-22T22:23:07.515 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-02-22T22:23:07.516 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-22T22:23:07.534 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libreadline-dev:amd64. 2026-02-22T22:23:07.536 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2026-02-22T22:23:07.542 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../019-libreadline-dev_8.1.2-1_amd64.deb ... 2026-02-22T22:23:07.542 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libreadline-dev:amd64 (8.1.2-1) ... 2026-02-22T22:23:07.544 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../004-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T22:23:07.545 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:07.559 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package liblua5.3-dev:amd64. 2026-02-22T22:23:07.561 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libqt5network5:amd64. 2026-02-22T22:23:07.567 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../020-liblua5.3-dev_5.3.6-1build1_amd64.deb ... 2026-02-22T22:23:07.567 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking liblua5.3-dev:amd64 (5.3.6-1build1) ... 2026-02-22T22:23:07.569 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../005-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2026-02-22T22:23:07.570 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:07.584 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package lua5.1. 2026-02-22T22:23:07.591 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2026-02-22T22:23:07.592 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../021-lua5.1_5.1.5-8.1build4_amd64.deb ... 2026-02-22T22:23:07.593 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking lua5.1 (5.1.5-8.1build4) ... 2026-02-22T22:23:07.599 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../006-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2026-02-22T22:23:07.599 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-22T22:23:07.611 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package lua-any. 2026-02-22T22:23:07.616 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package librados2. 2026-02-22T22:23:07.618 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../022-lua-any_27ubuntu1_all.deb ... 2026-02-22T22:23:07.619 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking lua-any (27ubuntu1) ... 2026-02-22T22:23:07.624 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../007-librados2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.624 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking librados2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.633 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package zip. 2026-02-22T22:23:07.641 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../023-zip_3.0-12build2_amd64.deb ... 2026-02-22T22:23:07.642 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking zip (3.0-12build2) ... 2026-02-22T22:23:07.658 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package unzip. 2026-02-22T22:23:07.665 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libnbd0. 2026-02-22T22:23:07.666 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../024-unzip_6.0-26ubuntu3.2_amd64.deb ... 2026-02-22T22:23:07.667 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking unzip (6.0-26ubuntu3.2) ... 2026-02-22T22:23:07.673 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../008-libnbd0_1.10.5-1_amd64.deb ... 2026-02-22T22:23:07.674 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libnbd0 (1.10.5-1) ... 2026-02-22T22:23:07.683 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package luarocks. 2026-02-22T22:23:07.688 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package librbd1. 2026-02-22T22:23:07.691 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../025-luarocks_3.8.0+dfsg1-1_all.deb ... 2026-02-22T22:23:07.692 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking luarocks (3.8.0+dfsg1-1) ... 2026-02-22T22:23:07.696 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../009-librbd1_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.697 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking librbd1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.729 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package librgw2. 2026-02-22T22:23:07.737 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../026-librgw2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.737 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking librgw2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.738 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libcephfs2. 2026-02-22T22:23:07.746 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../010-libcephfs2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.746 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libcephfs2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.767 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-rados. 2026-02-22T22:23:07.774 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../011-python3-rados_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.775 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-rados (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.792 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-ceph-argparse. 2026-02-22T22:23:07.800 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../012-python3-ceph-argparse_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:07.801 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-ceph-argparse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.810 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-rgw. 2026-02-22T22:23:07.814 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-cephfs. 2026-02-22T22:23:07.818 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../027-python3-rgw_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.818 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-rgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.822 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../013-python3-cephfs_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.823 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-cephfs (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.834 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package liboath0:amd64. 2026-02-22T22:23:07.838 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-ceph-common. 2026-02-22T22:23:07.841 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../028-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-02-22T22:23:07.842 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-22T22:23:07.846 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../014-python3-ceph-common_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:07.846 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.856 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libradosstriper1. 2026-02-22T22:23:07.863 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-wcwidth. 2026-02-22T22:23:07.864 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../029-libradosstriper1_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.865 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libradosstriper1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.871 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2026-02-22T22:23:07.871 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-22T22:23:07.884 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-common. 2026-02-22T22:23:07.887 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-prettytable. 2026-02-22T22:23:07.892 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../030-ceph-common_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.892 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.895 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2026-02-22T22:23:07.895 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2026-02-22T22:23:07.910 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-rbd. 2026-02-22T22:23:07.917 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../017-python3-rbd_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:07.918 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-rbd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:07.935 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package librdkafka1:amd64. 2026-02-22T22:23:07.943 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2026-02-22T22:23:07.944 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-22T22:23:07.961 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libreadline-dev:amd64. 2026-02-22T22:23:07.969 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../019-libreadline-dev_8.1.2-1_amd64.deb ... 2026-02-22T22:23:07.970 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libreadline-dev:amd64 (8.1.2-1) ... 2026-02-22T22:23:07.986 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package liblua5.3-dev:amd64. 2026-02-22T22:23:07.994 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../020-liblua5.3-dev_5.3.6-1build1_amd64.deb ... 2026-02-22T22:23:07.995 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking liblua5.3-dev:amd64 (5.3.6-1build1) ... 2026-02-22T22:23:08.012 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package lua5.1. 2026-02-22T22:23:08.019 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../021-lua5.1_5.1.5-8.1build4_amd64.deb ... 2026-02-22T22:23:08.020 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking lua5.1 (5.1.5-8.1build4) ... 2026-02-22T22:23:08.040 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package lua-any. 2026-02-22T22:23:08.048 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../022-lua-any_27ubuntu1_all.deb ... 2026-02-22T22:23:08.049 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking lua-any (27ubuntu1) ... 2026-02-22T22:23:08.063 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package zip. 2026-02-22T22:23:08.071 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../023-zip_3.0-12build2_amd64.deb ... 2026-02-22T22:23:08.072 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking zip (3.0-12build2) ... 2026-02-22T22:23:08.088 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package unzip. 2026-02-22T22:23:08.096 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../024-unzip_6.0-26ubuntu3.2_amd64.deb ... 2026-02-22T22:23:08.097 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking unzip (6.0-26ubuntu3.2) ... 2026-02-22T22:23:08.097 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-base. 2026-02-22T22:23:08.105 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../031-ceph-base_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:08.110 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-base (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.114 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package luarocks. 2026-02-22T22:23:08.122 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../025-luarocks_3.8.0+dfsg1-1_all.deb ... 2026-02-22T22:23:08.122 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking luarocks (3.8.0+dfsg1-1) ... 2026-02-22T22:23:08.160 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package librgw2. 2026-02-22T22:23:08.168 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../026-librgw2_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:08.169 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking librgw2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.173 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-02-22T22:23:08.181 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../032-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-02-22T22:23:08.182 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-02-22T22:23:08.196 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-cheroot. 2026-02-22T22:23:08.205 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../033-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-02-22T22:23:08.205 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-22T22:23:08.221 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-02-22T22:23:08.230 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../034-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-02-22T22:23:08.230 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-02-22T22:23:08.238 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-rgw. 2026-02-22T22:23:08.244 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-jaraco.text. 2026-02-22T22:23:08.246 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../027-python3-rgw_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:08.246 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-rgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.253 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../035-python3-jaraco.text_3.6.0-2_all.deb ... 2026-02-22T22:23:08.253 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-02-22T22:23:08.262 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package liboath0:amd64. 2026-02-22T22:23:08.268 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-02-22T22:23:08.270 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../028-liboath0_2.6.7-3ubuntu0.1_amd64.deb ... 2026-02-22T22:23:08.270 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-22T22:23:08.276 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../036-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-02-22T22:23:08.276 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-02-22T22:23:08.284 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libradosstriper1. 2026-02-22T22:23:08.290 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-tempora. 2026-02-22T22:23:08.292 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../029-libradosstriper1_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:08.293 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libradosstriper1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.298 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../037-python3-tempora_4.1.2-1_all.deb ... 2026-02-22T22:23:08.299 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-02-22T22:23:08.310 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-common. 2026-02-22T22:23:08.313 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-portend. 2026-02-22T22:23:08.318 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../030-ceph-common_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:08.319 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.322 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../038-python3-portend_3.0.0-1_all.deb ... 2026-02-22T22:23:08.322 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-02-22T22:23:08.336 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-02-22T22:23:08.344 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../039-python3-zc.lockfile_2.0-1_all.deb ... 2026-02-22T22:23:08.345 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-02-22T22:23:08.360 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-cherrypy3. 2026-02-22T22:23:08.368 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../040-python3-cherrypy3_18.6.1-4_all.deb ... 2026-02-22T22:23:08.368 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-02-22T22:23:08.391 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-natsort. 2026-02-22T22:23:08.399 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../041-python3-natsort_8.0.2-1_all.deb ... 2026-02-22T22:23:08.399 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-02-22T22:23:08.415 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-logutils. 2026-02-22T22:23:08.423 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../042-python3-logutils_0.3.3-8_all.deb ... 2026-02-22T22:23:08.423 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-logutils (0.3.3-8) ... 2026-02-22T22:23:08.438 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-mako. 2026-02-22T22:23:08.446 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../043-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2026-02-22T22:23:08.446 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2026-02-22T22:23:08.463 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-simplegeneric. 2026-02-22T22:23:08.471 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../044-python3-simplegeneric_0.8.1-3_all.deb ... 2026-02-22T22:23:08.472 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2026-02-22T22:23:08.486 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-singledispatch. 2026-02-22T22:23:08.494 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../045-python3-singledispatch_3.4.0.3-3_all.deb ... 2026-02-22T22:23:08.494 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2026-02-22T22:23:08.507 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-webob. 2026-02-22T22:23:08.515 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../046-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-02-22T22:23:08.516 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-22T22:23:08.520 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-base. 2026-02-22T22:23:08.528 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../031-ceph-base_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:08.532 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-waitress. 2026-02-22T22:23:08.533 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-base (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.540 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../047-python3-waitress_1.4.4-1.1ubuntu1.1_all.deb ... 2026-02-22T22:23:08.542 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1.1) ... 2026-02-22T22:23:08.557 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-tempita. 2026-02-22T22:23:08.565 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../048-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2026-02-22T22:23:08.566 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2026-02-22T22:23:08.580 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-paste. 2026-02-22T22:23:08.588 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../049-python3-paste_3.5.0+dfsg1-1_all.deb ... 2026-02-22T22:23:08.589 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2026-02-22T22:23:08.595 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-jaraco.functools. 2026-02-22T22:23:08.604 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../032-python3-jaraco.functools_3.4.0-2_all.deb ... 2026-02-22T22:23:08.604 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2026-02-22T22:23:08.616 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2026-02-22T22:23:08.618 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-cheroot. 2026-02-22T22:23:08.624 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../050-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2026-02-22T22:23:08.624 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2026-02-22T22:23:08.627 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../033-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2026-02-22T22:23:08.627 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-22T22:23:08.638 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-pastedeploy. 2026-02-22T22:23:08.644 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-jaraco.classes. 2026-02-22T22:23:08.646 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../051-python3-pastedeploy_2.1.1-1_all.deb ... 2026-02-22T22:23:08.647 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2026-02-22T22:23:08.652 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../034-python3-jaraco.classes_3.2.1-3_all.deb ... 2026-02-22T22:23:08.652 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2026-02-22T22:23:08.662 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-webtest. 2026-02-22T22:23:08.667 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-jaraco.text. 2026-02-22T22:23:08.670 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../052-python3-webtest_2.0.35-1_all.deb ... 2026-02-22T22:23:08.670 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-webtest (2.0.35-1) ... 2026-02-22T22:23:08.675 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../035-python3-jaraco.text_3.6.0-2_all.deb ... 2026-02-22T22:23:08.675 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2026-02-22T22:23:08.686 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-pecan. 2026-02-22T22:23:08.690 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-jaraco.collections. 2026-02-22T22:23:08.694 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../053-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2026-02-22T22:23:08.694 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2026-02-22T22:23:08.698 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../036-python3-jaraco.collections_3.4.0-2_all.deb ... 2026-02-22T22:23:08.699 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2026-02-22T22:23:08.713 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-tempora. 2026-02-22T22:23:08.719 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-werkzeug. 2026-02-22T22:23:08.721 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../037-python3-tempora_4.1.2-1_all.deb ... 2026-02-22T22:23:08.721 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-tempora (4.1.2-1) ... 2026-02-22T22:23:08.727 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../054-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.3_all.deb ... 2026-02-22T22:23:08.728 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.3) ... 2026-02-22T22:23:08.736 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-portend. 2026-02-22T22:23:08.745 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../038-python3-portend_3.0.0-1_all.deb ... 2026-02-22T22:23:08.745 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-portend (3.0.0-1) ... 2026-02-22T22:23:08.747 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-02-22T22:23:08.755 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../055-ceph-mgr-modules-core_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:08.755 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-mgr-modules-core (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.759 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-zc.lockfile. 2026-02-22T22:23:08.767 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../039-python3-zc.lockfile_2.0-1_all.deb ... 2026-02-22T22:23:08.767 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2026-02-22T22:23:08.782 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-cherrypy3. 2026-02-22T22:23:08.782 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-02-22T22:23:08.790 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../040-python3-cherrypy3_18.6.1-4_all.deb ... 2026-02-22T22:23:08.790 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:08.790 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2026-02-22T22:23:08.790 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libsqlite3-mod-ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.805 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-mgr. 2026-02-22T22:23:08.812 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-natsort. 2026-02-22T22:23:08.813 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../057-ceph-mgr_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:08.814 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-mgr (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.820 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../041-python3-natsort_8.0.2-1_all.deb ... 2026-02-22T22:23:08.820 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-natsort (8.0.2-1) ... 2026-02-22T22:23:08.835 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-logutils. 2026-02-22T22:23:08.836 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-mon. 2026-02-22T22:23:08.843 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../042-python3-logutils_0.3.3-8_all.deb ... 2026-02-22T22:23:08.844 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-logutils (0.3.3-8) ... 2026-02-22T22:23:08.844 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../058-ceph-mon_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:08.844 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-mon (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.858 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-mako. 2026-02-22T22:23:08.866 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../043-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2026-02-22T22:23:08.867 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2026-02-22T22:23:08.884 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-simplegeneric. 2026-02-22T22:23:08.892 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../044-python3-simplegeneric_0.8.1-3_all.deb ... 2026-02-22T22:23:08.893 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2026-02-22T22:23:08.903 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libfuse2:amd64. 2026-02-22T22:23:08.907 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-singledispatch. 2026-02-22T22:23:08.911 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../059-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-02-22T22:23:08.911 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-22T22:23:08.915 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../045-python3-singledispatch_3.4.0.3-3_all.deb ... 2026-02-22T22:23:08.915 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2026-02-22T22:23:08.927 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-osd. 2026-02-22T22:23:08.928 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-webob. 2026-02-22T22:23:08.935 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../060-ceph-osd_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:08.936 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-osd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:08.936 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../046-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2026-02-22T22:23:08.937 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-22T22:23:08.953 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-waitress. 2026-02-22T22:23:08.961 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../047-python3-waitress_1.4.4-1.1ubuntu1.1_all.deb ... 2026-02-22T22:23:08.963 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1.1) ... 2026-02-22T22:23:08.979 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-tempita. 2026-02-22T22:23:08.987 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../048-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2026-02-22T22:23:08.987 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2026-02-22T22:23:09.000 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-paste. 2026-02-22T22:23:09.008 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../049-python3-paste_3.5.0+dfsg1-1_all.deb ... 2026-02-22T22:23:09.009 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2026-02-22T22:23:09.036 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2026-02-22T22:23:09.045 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../050-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2026-02-22T22:23:09.045 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2026-02-22T22:23:09.059 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-pastedeploy. 2026-02-22T22:23:09.067 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../051-python3-pastedeploy_2.1.1-1_all.deb ... 2026-02-22T22:23:09.067 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2026-02-22T22:23:09.083 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-webtest. 2026-02-22T22:23:09.091 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../052-python3-webtest_2.0.35-1_all.deb ... 2026-02-22T22:23:09.091 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph. 2026-02-22T22:23:09.091 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-webtest (2.0.35-1) ... 2026-02-22T22:23:09.100 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../061-ceph_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.100 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.106 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-pecan. 2026-02-22T22:23:09.114 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-fuse. 2026-02-22T22:23:09.114 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../053-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2026-02-22T22:23:09.115 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2026-02-22T22:23:09.122 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../062-ceph-fuse_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.122 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.139 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-werkzeug. 2026-02-22T22:23:09.146 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-mds. 2026-02-22T22:23:09.147 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../054-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.3_all.deb ... 2026-02-22T22:23:09.147 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.3) ... 2026-02-22T22:23:09.154 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../063-ceph-mds_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.155 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-mds (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.167 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2026-02-22T22:23:09.174 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../055-ceph-mgr-modules-core_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:09.175 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-mgr-modules-core (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.188 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package cephadm. 2026-02-22T22:23:09.196 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../064-cephadm_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.196 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.203 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2026-02-22T22:23:09.211 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.211 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libsqlite3-mod-ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.213 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-asyncssh. 2026-02-22T22:23:09.221 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../065-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-02-22T22:23:09.221 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-22T22:23:09.226 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-mgr. 2026-02-22T22:23:09.234 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../057-ceph-mgr_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.235 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-mgr (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.243 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-02-22T22:23:09.251 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../066-ceph-mgr-cephadm_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:09.252 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-mgr-cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.257 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-mon. 2026-02-22T22:23:09.265 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../058-ceph-mon_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.266 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-mon (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.271 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-repoze.lru. 2026-02-22T22:23:09.279 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../067-python3-repoze.lru_0.7-2_all.deb ... 2026-02-22T22:23:09.280 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-02-22T22:23:09.295 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-routes. 2026-02-22T22:23:09.303 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../068-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-02-22T22:23:09.303 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-02-22T22:23:09.323 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libfuse2:amd64. 2026-02-22T22:23:09.323 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-02-22T22:23:09.331 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../059-libfuse2_2.9.9-5ubuntu3_amd64.deb ... 2026-02-22T22:23:09.332 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-22T22:23:09.332 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../069-ceph-mgr-dashboard_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:09.332 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-mgr-dashboard (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.347 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-osd. 2026-02-22T22:23:09.356 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../060-ceph-osd_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.356 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-osd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.510 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph. 2026-02-22T22:23:09.518 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../061-ceph_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.519 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.532 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-fuse. 2026-02-22T22:23:09.534 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-02-22T22:23:09.540 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../062-ceph-fuse_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.540 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.542 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../070-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-02-22T22:23:09.542 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-22T22:23:09.564 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-mds. 2026-02-22T22:23:09.572 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../063-ceph-mds_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.573 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-mds (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.581 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-joblib. 2026-02-22T22:23:09.589 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../071-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-02-22T22:23:09.590 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-02-22T22:23:09.606 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package cephadm. 2026-02-22T22:23:09.614 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../064-cephadm_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:09.615 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.616 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-02-22T22:23:09.624 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../072-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-02-22T22:23:09.625 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-02-22T22:23:09.631 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-asyncssh. 2026-02-22T22:23:09.639 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../065-python3-asyncssh_2.5.0-1ubuntu0.1_all.deb ... 2026-02-22T22:23:09.640 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-22T22:23:09.640 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-sklearn. 2026-02-22T22:23:09.648 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../073-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-02-22T22:23:09.649 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-22T22:23:09.660 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2026-02-22T22:23:09.668 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../066-ceph-mgr-cephadm_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:09.669 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-mgr-cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.688 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-repoze.lru. 2026-02-22T22:23:09.697 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../067-python3-repoze.lru_0.7-2_all.deb ... 2026-02-22T22:23:09.697 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2026-02-22T22:23:09.712 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-routes. 2026-02-22T22:23:09.720 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../068-python3-routes_2.5.1-1ubuntu1_all.deb ... 2026-02-22T22:23:09.720 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2026-02-22T22:23:09.732 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-02-22T22:23:09.740 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2026-02-22T22:23:09.740 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../074-ceph-mgr-diskprediction-local_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:09.741 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-mgr-diskprediction-local (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.748 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../069-ceph-mgr-dashboard_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:09.748 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-mgr-dashboard (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:09.910 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-cachetools. 2026-02-22T22:23:09.918 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../075-python3-cachetools_5.0.0-1_all.deb ... 2026-02-22T22:23:09.919 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-02-22T22:23:09.933 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-rsa. 2026-02-22T22:23:09.940 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../076-python3-rsa_4.8-1_all.deb ... 2026-02-22T22:23:09.940 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-rsa (4.8-1) ... 2026-02-22T22:23:09.952 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2026-02-22T22:23:09.958 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-google-auth. 2026-02-22T22:23:09.961 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../070-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2026-02-22T22:23:09.961 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-22T22:23:09.966 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../077-python3-google-auth_1.5.1-3_all.deb ... 2026-02-22T22:23:09.966 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-02-22T22:23:09.983 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-02-22T22:23:09.992 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../078-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-02-22T22:23:09.993 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-22T22:23:10.000 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-joblib. 2026-02-22T22:23:10.008 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-websocket. 2026-02-22T22:23:10.008 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../071-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2026-02-22T22:23:10.009 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2026-02-22T22:23:10.015 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../079-python3-websocket_1.2.3-1_all.deb ... 2026-02-22T22:23:10.015 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-02-22T22:23:10.032 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-kubernetes. 2026-02-22T22:23:10.036 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-threadpoolctl. 2026-02-22T22:23:10.040 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../080-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-02-22T22:23:10.041 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-22T22:23:10.045 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../072-python3-threadpoolctl_3.1.0-1_all.deb ... 2026-02-22T22:23:10.045 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2026-02-22T22:23:10.060 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-sklearn. 2026-02-22T22:23:10.069 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../073-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2026-02-22T22:23:10.069 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-22T22:23:10.136 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-02-22T22:23:10.145 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../081-ceph-mgr-k8sevents_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:10.145 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-mgr-k8sevents (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:10.148 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2026-02-22T22:23:10.158 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../074-ceph-mgr-diskprediction-local_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:10.158 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-mgr-diskprediction-local (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:10.159 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libonig5:amd64. 2026-02-22T22:23:10.167 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../082-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-02-22T22:23:10.175 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-22T22:23:10.190 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libjq1:amd64. 2026-02-22T22:23:10.199 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../083-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-22T22:23:10.199 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-22T22:23:10.215 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package jq. 2026-02-22T22:23:10.223 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../084-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-22T22:23:10.224 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-02-22T22:23:10.237 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package socat. 2026-02-22T22:23:10.246 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../085-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-02-22T22:23:10.246 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-02-22T22:23:10.266 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package xmlstarlet. 2026-02-22T22:23:10.274 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../086-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-02-22T22:23:10.275 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-02-22T22:23:10.311 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-test. 2026-02-22T22:23:10.320 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../087-ceph-test_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:10.320 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-test (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:10.322 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-cachetools. 2026-02-22T22:23:10.331 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../075-python3-cachetools_5.0.0-1_all.deb ... 2026-02-22T22:23:10.331 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2026-02-22T22:23:10.346 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-rsa. 2026-02-22T22:23:10.354 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../076-python3-rsa_4.8-1_all.deb ... 2026-02-22T22:23:10.355 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-rsa (4.8-1) ... 2026-02-22T22:23:10.371 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-google-auth. 2026-02-22T22:23:10.379 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../077-python3-google-auth_1.5.1-3_all.deb ... 2026-02-22T22:23:10.380 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2026-02-22T22:23:10.397 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-requests-oauthlib. 2026-02-22T22:23:10.405 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../078-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2026-02-22T22:23:10.406 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-22T22:23:10.422 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-websocket. 2026-02-22T22:23:10.430 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../079-python3-websocket_1.2.3-1_all.deb ... 2026-02-22T22:23:10.431 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-websocket (1.2.3-1) ... 2026-02-22T22:23:10.448 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-kubernetes. 2026-02-22T22:23:10.456 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../080-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2026-02-22T22:23:10.456 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-22T22:23:10.551 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2026-02-22T22:23:10.560 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../081-ceph-mgr-k8sevents_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:10.560 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-mgr-k8sevents (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:10.574 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libonig5:amd64. 2026-02-22T22:23:10.583 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../082-libonig5_6.9.7.1-2build1_amd64.deb ... 2026-02-22T22:23:10.590 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-22T22:23:10.605 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libjq1:amd64. 2026-02-22T22:23:10.613 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../083-libjq1_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-22T22:23:10.614 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-22T22:23:10.628 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package jq. 2026-02-22T22:23:10.636 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../084-jq_1.6-2.1ubuntu3.1_amd64.deb ... 2026-02-22T22:23:10.637 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking jq (1.6-2.1ubuntu3.1) ... 2026-02-22T22:23:10.651 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package socat. 2026-02-22T22:23:10.659 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../085-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2026-02-22T22:23:10.660 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2026-02-22T22:23:10.679 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package xmlstarlet. 2026-02-22T22:23:10.687 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../086-xmlstarlet_1.6.1-2.1_amd64.deb ... 2026-02-22T22:23:10.688 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2026-02-22T22:23:10.724 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-test. 2026-02-22T22:23:10.733 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../087-ceph-test_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:10.733 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-test (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:10.802 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package ceph-volume. 2026-02-22T22:23:10.811 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../088-ceph-volume_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:10.812 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking ceph-volume (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:10.834 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package libcephfs-dev. 2026-02-22T22:23:10.842 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../089-libcephfs-dev_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:10.842 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking libcephfs-dev (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:10.857 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package lua-socket:amd64. 2026-02-22T22:23:10.866 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../090-lua-socket_3.0~rc1+git+ac3201d-6_amd64.deb ... 2026-02-22T22:23:10.866 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2026-02-22T22:23:10.888 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package lua-sec:amd64. 2026-02-22T22:23:10.896 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../091-lua-sec_1.0.2-1_amd64.deb ... 2026-02-22T22:23:10.896 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking lua-sec:amd64 (1.0.2-1) ... 2026-02-22T22:23:10.914 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package nvme-cli. 2026-02-22T22:23:10.922 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../092-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-02-22T22:23:10.923 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-02-22T22:23:10.954 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package pkg-config. 2026-02-22T22:23:10.962 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../093-pkg-config_0.29.2-1ubuntu3_amd64.deb ... 2026-02-22T22:23:10.963 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking pkg-config (0.29.2-1ubuntu3) ... 2026-02-22T22:23:10.978 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-02-22T22:23:10.987 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../094-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-02-22T22:23:10.987 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-22T22:23:11.016 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-iniconfig. 2026-02-22T22:23:11.025 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../095-python3-iniconfig_1.1.1-2_all.deb ... 2026-02-22T22:23:11.025 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-02-22T22:23:11.040 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-pastescript. 2026-02-22T22:23:11.048 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../096-python3-pastescript_2.0.2-4_all.deb ... 2026-02-22T22:23:11.049 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2026-02-22T22:23:11.067 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-pluggy. 2026-02-22T22:23:11.076 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../097-python3-pluggy_0.13.0-7.1_all.deb ... 2026-02-22T22:23:11.077 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-02-22T22:23:11.092 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-psutil. 2026-02-22T22:23:11.101 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../098-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-02-22T22:23:11.101 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-02-22T22:23:11.119 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-py. 2026-02-22T22:23:11.128 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../099-python3-py_1.10.0-1_all.deb ... 2026-02-22T22:23:11.128 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-py (1.10.0-1) ... 2026-02-22T22:23:11.149 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-pygments. 2026-02-22T22:23:11.157 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../100-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-02-22T22:23:11.158 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-22T22:23:11.199 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-pyinotify. 2026-02-22T22:23:11.203 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package ceph-volume. 2026-02-22T22:23:11.208 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../101-python3-pyinotify_0.9.6-1.3_all.deb ... 2026-02-22T22:23:11.208 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2026-02-22T22:23:11.212 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../088-ceph-volume_19.2.3-689-gf90f4eae-1jammy_all.deb ... 2026-02-22T22:23:11.212 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking ceph-volume (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:11.223 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-toml. 2026-02-22T22:23:11.231 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../102-python3-toml_0.10.2-1_all.deb ... 2026-02-22T22:23:11.232 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-02-22T22:23:11.235 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package libcephfs-dev. 2026-02-22T22:23:11.243 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../089-libcephfs-dev_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:11.244 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking libcephfs-dev (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:11.247 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-pytest. 2026-02-22T22:23:11.255 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../103-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-02-22T22:23:11.256 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-02-22T22:23:11.259 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package lua-socket:amd64. 2026-02-22T22:23:11.268 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../090-lua-socket_3.0~rc1+git+ac3201d-6_amd64.deb ... 2026-02-22T22:23:11.268 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2026-02-22T22:23:11.277 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package python3-simplejson. 2026-02-22T22:23:11.285 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../104-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-02-22T22:23:11.286 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-02-22T22:23:11.289 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package lua-sec:amd64. 2026-02-22T22:23:11.297 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../091-lua-sec_1.0.2-1_amd64.deb ... 2026-02-22T22:23:11.297 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking lua-sec:amd64 (1.0.2-1) ... 2026-02-22T22:23:11.303 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-02-22T22:23:11.311 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../105-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-02-22T22:23:11.312 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-02-22T22:23:11.314 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package nvme-cli. 2026-02-22T22:23:11.322 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../092-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2026-02-22T22:23:11.323 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2026-02-22T22:23:11.354 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package pkg-config. 2026-02-22T22:23:11.362 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../093-pkg-config_0.29.2-1ubuntu3_amd64.deb ... 2026-02-22T22:23:11.362 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking pkg-config (0.29.2-1ubuntu3) ... 2026-02-22T22:23:11.378 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package radosgw. 2026-02-22T22:23:11.378 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python-asyncssh-doc. 2026-02-22T22:23:11.386 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../106-radosgw_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:11.388 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking radosgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:11.388 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../094-python-asyncssh-doc_2.5.0-1ubuntu0.1_all.deb ... 2026-02-22T22:23:11.388 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-22T22:23:11.417 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-iniconfig. 2026-02-22T22:23:11.425 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../095-python3-iniconfig_1.1.1-2_all.deb ... 2026-02-22T22:23:11.426 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2026-02-22T22:23:11.441 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-pastescript. 2026-02-22T22:23:11.449 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../096-python3-pastescript_2.0.2-4_all.deb ... 2026-02-22T22:23:11.449 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2026-02-22T22:23:11.468 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-pluggy. 2026-02-22T22:23:11.477 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../097-python3-pluggy_0.13.0-7.1_all.deb ... 2026-02-22T22:23:11.477 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2026-02-22T22:23:11.493 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-psutil. 2026-02-22T22:23:11.501 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../098-python3-psutil_5.9.0-1build1_amd64.deb ... 2026-02-22T22:23:11.501 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2026-02-22T22:23:11.503 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package rbd-fuse. 2026-02-22T22:23:11.512 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../107-rbd-fuse_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:11.513 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking rbd-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:11.520 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-py. 2026-02-22T22:23:11.528 INFO:teuthology.orchestra.run.trial190.stdout:Selecting previously unselected package rbd-mirror. 2026-02-22T22:23:11.528 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../099-python3-py_1.10.0-1_all.deb ... 2026-02-22T22:23:11.528 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-py (1.10.0-1) ... 2026-02-22T22:23:11.536 INFO:teuthology.orchestra.run.trial190.stdout:Preparing to unpack .../108-rbd-mirror_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:11.536 INFO:teuthology.orchestra.run.trial190.stdout:Unpacking rbd-mirror (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:11.548 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-pygments. 2026-02-22T22:23:11.556 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../100-python3-pygments_2.11.2+dfsg-2ubuntu0.1_all.deb ... 2026-02-22T22:23:11.557 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-22T22:23:11.596 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-02-22T22:23:11.597 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-pyinotify. 2026-02-22T22:23:11.606 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../101-python3-pyinotify_0.9.6-1.3_all.deb ... 2026-02-22T22:23:11.606 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2026-02-22T22:23:11.621 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-toml. 2026-02-22T22:23:11.630 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../102-python3-toml_0.10.2-1_all.deb ... 2026-02-22T22:23:11.630 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-toml (0.10.2-1) ... 2026-02-22T22:23:11.645 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-pytest. 2026-02-22T22:23:11.653 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../103-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2026-02-22T22:23:11.654 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2026-02-22T22:23:11.661 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-22T22:23:11.662 INFO:teuthology.orchestra.run.trial190.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-02-22T22:23:11.676 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package python3-simplejson. 2026-02-22T22:23:11.684 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../104-python3-simplejson_3.17.6-1build1_amd64.deb ... 2026-02-22T22:23:11.685 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2026-02-22T22:23:11.701 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package qttranslations5-l10n. 2026-02-22T22:23:11.710 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../105-qttranslations5-l10n_5.15.3-1_all.deb ... 2026-02-22T22:23:11.710 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2026-02-22T22:23:11.733 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T22:23:11.776 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package radosgw. 2026-02-22T22:23:11.784 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../106-radosgw_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:11.785 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking radosgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:11.899 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package rbd-fuse. 2026-02-22T22:23:11.908 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../107-rbd-fuse_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:11.909 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking rbd-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:11.924 INFO:teuthology.orchestra.run.trial103.stdout:Selecting previously unselected package rbd-mirror. 2026-02-22T22:23:11.933 INFO:teuthology.orchestra.run.trial103.stdout:Preparing to unpack .../108-rbd-mirror_19.2.3-689-gf90f4eae-1jammy_amd64.deb ... 2026-02-22T22:23:11.933 INFO:teuthology.orchestra.run.trial103.stdout:Unpacking rbd-mirror (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:11.992 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2026-02-22T22:23:12.013 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-02-22T22:23:12.062 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2026-02-22T22:23:12.063 INFO:teuthology.orchestra.run.trial103.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2026-02-22T22:23:12.127 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T22:23:12.358 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service → /lib/systemd/system/nvmf-autoconnect.service. 2026-02-22T22:23:12.380 INFO:teuthology.orchestra.run.trial190.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-02-22T22:23:12.390 INFO:teuthology.orchestra.run.trial190.stdout:Setting up cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:12.432 INFO:teuthology.orchestra.run.trial190.stdout:Adding system user cephadm....done 2026-02-22T22:23:12.443 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1.1) ... 2026-02-22T22:23:12.515 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-02-22T22:23:12.586 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-22T22:23:12.587 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-02-22T22:23:12.661 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-02-22T22:23:12.708 INFO:teuthology.orchestra.run.trial103.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2026-02-22T22:23:12.719 INFO:teuthology.orchestra.run.trial103.stdout:Setting up cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:12.737 INFO:teuthology.orchestra.run.trial190.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-22T22:23:12.738 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-py (1.10.0-1) ... 2026-02-22T22:23:12.763 INFO:teuthology.orchestra.run.trial103.stdout:Adding system user cephadm....done 2026-02-22T22:23:12.774 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1.1) ... 2026-02-22T22:23:12.838 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-02-22T22:23:12.843 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2026-02-22T22:23:12.905 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python-asyncssh-doc (2.5.0-1ubuntu0.1) ... 2026-02-22T22:23:12.906 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2026-02-22T22:23:12.962 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-02-22T22:23:12.970 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-repoze.lru (0.7-2) ... 2026-02-22T22:23:13.037 INFO:teuthology.orchestra.run.trial190.stdout:Setting up unzip (6.0-26ubuntu3.2) ... 2026-02-22T22:23:13.046 INFO:teuthology.orchestra.run.trial103.stdout:Setting up liboath0:amd64 (2.6.7-3ubuntu0.1) ... 2026-02-22T22:23:13.047 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-py (1.10.0-1) ... 2026-02-22T22:23:13.048 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2026-02-22T22:23:13.126 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-02-22T22:23:13.132 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2026-02-22T22:23:13.199 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-ceph-argparse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:13.257 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-cachetools (5.0.0-1) ... 2026-02-22T22:23:13.267 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-22T22:23:13.268 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-02-22T22:23:13.269 INFO:teuthology.orchestra.run.trial190.stdout:Setting up lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2026-02-22T22:23:13.270 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libreadline-dev:amd64 (8.1.2-1) ... 2026-02-22T22:23:13.271 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-22T22:23:13.272 INFO:teuthology.orchestra.run.trial190.stdout:Setting up lua5.1 (5.1.5-8.1build4) ... 2026-02-22T22:23:13.289 INFO:teuthology.orchestra.run.trial190.stdout:update-alternatives: using /usr/bin/lua5.1 to provide /usr/bin/lua (lua-interpreter) in auto mode 2026-02-22T22:23:13.291 INFO:teuthology.orchestra.run.trial190.stdout:update-alternatives: using /usr/bin/luac5.1 to provide /usr/bin/luac (lua-compiler) in auto mode 2026-02-22T22:23:13.292 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-22T22:23:13.293 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-02-22T22:23:13.324 INFO:teuthology.orchestra.run.trial103.stdout:Setting up unzip (6.0-26ubuntu3.2) ... 2026-02-22T22:23:13.336 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2026-02-22T22:23:13.403 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2026-02-22T22:23:13.419 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-02-22T22:23:13.477 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-ceph-argparse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:13.496 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-02-22T22:23:13.544 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2026-02-22T22:23:13.545 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libnbd0 (1.10.5-1) ... 2026-02-22T22:23:13.546 INFO:teuthology.orchestra.run.trial103.stdout:Setting up lua-socket:amd64 (3.0~rc1+git+ac3201d-6) ... 2026-02-22T22:23:13.547 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libreadline-dev:amd64 (8.1.2-1) ... 2026-02-22T22:23:13.548 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libfuse2:amd64 (2.9.9-5ubuntu3) ... 2026-02-22T22:23:13.549 INFO:teuthology.orchestra.run.trial103.stdout:Setting up lua5.1 (5.1.5-8.1build4) ... 2026-02-22T22:23:13.566 INFO:teuthology.orchestra.run.trial103.stdout:update-alternatives: using /usr/bin/lua5.1 to provide /usr/bin/lua (lua-interpreter) in auto mode 2026-02-22T22:23:13.567 INFO:teuthology.orchestra.run.trial103.stdout:update-alternatives: using /usr/bin/luac5.1 to provide /usr/bin/luac (lua-compiler) in auto mode 2026-02-22T22:23:13.568 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2026-02-22T22:23:13.569 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2026-02-22T22:23:13.576 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-02-22T22:23:13.668 INFO:teuthology.orchestra.run.trial190.stdout:Setting up zip (3.0-12build2) ... 2026-02-22T22:23:13.669 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-22T22:23:13.683 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-natsort (8.0.2-1) ... 2026-02-22T22:23:13.752 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2026-02-22T22:23:13.832 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2026-02-22T22:23:13.911 INFO:teuthology.orchestra.run.trial103.stdout:Setting up zip (3.0-12build2) ... 2026-02-22T22:23:13.912 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-pygments (2.11.2+dfsg-2ubuntu0.1) ... 2026-02-22T22:23:13.928 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2026-02-22T22:23:14.006 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2026-02-22T22:23:14.006 INFO:teuthology.orchestra.run.trial190.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-02-22T22:23:14.007 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-22T22:23:14.105 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-22T22:23:14.164 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2026-02-22T22:23:14.230 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2026-02-22T22:23:14.242 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2026-02-22T22:23:14.243 INFO:teuthology.orchestra.run.trial103.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2026-02-22T22:23:14.244 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2026-02-22T22:23:14.336 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-asyncssh (2.5.0-1ubuntu0.1) ... 2026-02-22T22:23:14.359 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-22T22:23:14.440 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.3) ... 2026-02-22T22:23:14.458 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2026-02-22T22:23:14.547 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-02-22T22:23:14.587 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2026-02-22T22:23:14.622 INFO:teuthology.orchestra.run.trial190.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-02-22T22:23:14.623 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:14.667 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.3) ... 2026-02-22T22:23:14.707 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-22T22:23:14.772 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2026-02-22T22:23:14.843 INFO:teuthology.orchestra.run.trial103.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2026-02-22T22:23:14.843 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:14.925 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2026-02-22T22:23:15.219 INFO:teuthology.orchestra.run.trial190.stdout:Setting up pkg-config (0.29.2-1ubuntu3) ... 2026-02-22T22:23:15.242 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:15.246 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-toml (0.10.2-1) ... 2026-02-22T22:23:15.322 INFO:teuthology.orchestra.run.trial190.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-22T22:23:15.323 INFO:teuthology.orchestra.run.trial190.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-02-22T22:23:15.324 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-02-22T22:23:15.400 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-02-22T22:23:15.420 INFO:teuthology.orchestra.run.trial103.stdout:Setting up pkg-config (0.29.2-1ubuntu3) ... 2026-02-22T22:23:15.442 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:15.447 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-toml (0.10.2-1) ... 2026-02-22T22:23:15.473 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:15.474 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-rsa (4.8-1) ... 2026-02-22T22:23:15.523 INFO:teuthology.orchestra.run.trial103.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2026-02-22T22:23:15.524 INFO:teuthology.orchestra.run.trial103.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2026-02-22T22:23:15.525 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2026-02-22T22:23:15.551 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2026-02-22T22:23:15.602 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2026-02-22T22:23:15.623 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-logutils (0.3.3-8) ... 2026-02-22T22:23:15.677 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:15.678 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-rsa (4.8-1) ... 2026-02-22T22:23:15.700 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-02-22T22:23:15.757 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2026-02-22T22:23:15.776 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2026-02-22T22:23:15.831 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-logutils (0.3.3-8) ... 2026-02-22T22:23:15.849 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-02-22T22:23:15.911 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-tempora (4.1.2-1) ... 2026-02-22T22:23:15.931 INFO:teuthology.orchestra.run.trial190.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-22T22:23:15.932 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-02-22T22:23:15.984 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2026-02-22T22:23:16.016 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-22T22:23:16.017 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-22T22:23:16.061 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-prettytable (2.5.0-2) ... 2026-02-22T22:23:16.097 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2026-02-22T22:23:16.140 INFO:teuthology.orchestra.run.trial103.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2026-02-22T22:23:16.141 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-websocket (1.2.3-1) ... 2026-02-22T22:23:16.177 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-22T22:23:16.225 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2026-02-22T22:23:16.226 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2026-02-22T22:23:16.265 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-02-22T22:23:16.306 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2026-02-22T22:23:16.339 INFO:teuthology.orchestra.run.trial190.stdout:Setting up liblua5.3-dev:amd64 (5.3.6-1build1) ... 2026-02-22T22:23:16.340 INFO:teuthology.orchestra.run.trial190.stdout:Setting up lua-sec:amd64 (1.0.2-1) ... 2026-02-22T22:23:16.341 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-22T22:23:16.342 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-02-22T22:23:16.385 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2026-02-22T22:23:16.469 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2026-02-22T22:23:16.480 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2026-02-22T22:23:16.544 INFO:teuthology.orchestra.run.trial103.stdout:Setting up liblua5.3-dev:amd64 (5.3.6-1build1) ... 2026-02-22T22:23:16.545 INFO:teuthology.orchestra.run.trial103.stdout:Setting up lua-sec:amd64 (1.0.2-1) ... 2026-02-22T22:23:16.546 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3.1) ... 2026-02-22T22:23:16.546 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2026-02-22T22:23:16.557 INFO:teuthology.orchestra.run.trial190.stdout:Setting up lua-any (27ubuntu1) ... 2026-02-22T22:23:16.558 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-portend (3.0.0-1) ... 2026-02-22T22:23:16.630 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:16.631 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-02-22T22:23:16.686 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2026-02-22T22:23:16.715 INFO:teuthology.orchestra.run.trial190.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-02-22T22:23:16.716 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-webtest (2.0.35-1) ... 2026-02-22T22:23:16.765 INFO:teuthology.orchestra.run.trial103.stdout:Setting up lua-any (27ubuntu1) ... 2026-02-22T22:23:16.767 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-portend (3.0.0-1) ... 2026-02-22T22:23:16.798 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-02-22T22:23:16.839 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2026-02-22T22:23:16.840 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-google-auth (1.5.1-3) ... 2026-02-22T22:23:16.926 INFO:teuthology.orchestra.run.trial103.stdout:Setting up jq (1.6-2.1ubuntu3.1) ... 2026-02-22T22:23:16.927 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-webtest (2.0.35-1) ... 2026-02-22T22:23:16.929 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-pastescript (2.0.2-4) ... 2026-02-22T22:23:17.009 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2026-02-22T22:23:17.020 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2026-02-22T22:23:17.137 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-22T22:23:17.138 INFO:teuthology.orchestra.run.trial190.stdout:Setting up librados2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.139 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libsqlite3-mod-ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.140 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-22T22:23:17.141 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-pastescript (2.0.2-4) ... 2026-02-22T22:23:17.231 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2026-02-22T22:23:17.345 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2026-02-22T22:23:17.346 INFO:teuthology.orchestra.run.trial103.stdout:Setting up librados2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.347 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libsqlite3-mod-ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.348 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2026-02-22T22:23:17.677 INFO:teuthology.orchestra.run.trial190.stdout:Setting up luarocks (3.8.0+dfsg1-1) ... 2026-02-22T22:23:17.688 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libcephfs2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.688 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libradosstriper1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.689 INFO:teuthology.orchestra.run.trial190.stdout:Setting up librbd1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.690 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-mgr-modules-core (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.691 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.763 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-22T22:23:17.763 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-22T22:23:17.867 INFO:teuthology.orchestra.run.trial103.stdout:Setting up luarocks (3.8.0+dfsg1-1) ... 2026-02-22T22:23:17.879 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libcephfs2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.880 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libradosstriper1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.881 INFO:teuthology.orchestra.run.trial103.stdout:Setting up librbd1 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.881 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-mgr-modules-core (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.882 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:17.958 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-22T22:23:17.958 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target → /lib/systemd/system/ceph-fuse.target. 2026-02-22T22:23:18.172 INFO:teuthology.orchestra.run.trial190.stdout:Setting up libcephfs-dev (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.173 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-rados (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.173 INFO:teuthology.orchestra.run.trial190.stdout:Setting up librgw2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.174 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-rbd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.175 INFO:teuthology.orchestra.run.trial190.stdout:Setting up rbd-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.176 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-rgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.177 INFO:teuthology.orchestra.run.trial190.stdout:Setting up python3-cephfs (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.178 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.206 INFO:teuthology.orchestra.run.trial190.stdout:Adding group ceph....done 2026-02-22T22:23:18.240 INFO:teuthology.orchestra.run.trial190.stdout:Adding system user ceph....done 2026-02-22T22:23:18.249 INFO:teuthology.orchestra.run.trial190.stdout:Setting system user ceph properties....done 2026-02-22T22:23:18.256 INFO:teuthology.orchestra.run.trial190.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-02-22T22:23:18.304 INFO:teuthology.orchestra.run.trial103.stdout:Setting up libcephfs-dev (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.305 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-rados (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.306 INFO:teuthology.orchestra.run.trial103.stdout:Setting up librgw2 (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.307 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-rbd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.308 INFO:teuthology.orchestra.run.trial103.stdout:Setting up rbd-fuse (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.309 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-rgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.309 INFO:teuthology.orchestra.run.trial103.stdout:Setting up python3-cephfs (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.310 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-common (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:18.339 INFO:teuthology.orchestra.run.trial103.stdout:Adding group ceph....done 2026-02-22T22:23:18.348 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-02-22T22:23:18.374 INFO:teuthology.orchestra.run.trial103.stdout:Adding system user ceph....done 2026-02-22T22:23:18.385 INFO:teuthology.orchestra.run.trial103.stdout:Setting system user ceph properties....done 2026-02-22T22:23:18.392 INFO:teuthology.orchestra.run.trial103.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2026-02-22T22:23:18.480 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /lib/systemd/system/ceph.target. 2026-02-22T22:23:18.637 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-02-22T22:23:18.746 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service → /lib/systemd/system/rbdmap.service. 2026-02-22T22:23:19.014 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-test (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:19.015 INFO:teuthology.orchestra.run.trial190.stdout:Setting up radosgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:19.142 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-test (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:19.143 INFO:teuthology.orchestra.run.trial103.stdout:Setting up radosgw (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:19.255 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:23:19.255 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:23:19.389 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:23:19.389 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:23:19.603 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-base (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:19.717 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-02-22T22:23:19.732 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-base (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:19.834 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /lib/systemd/system/ceph-crash.service. 2026-02-22T22:23:20.073 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-mds (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:20.133 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-22T22:23:20.133 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-22T22:23:20.182 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-mds (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:20.243 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-22T22:23:20.243 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /lib/systemd/system/ceph-mds.target. 2026-02-22T22:23:20.476 INFO:teuthology.orchestra.run.trial190.stdout:Setting up rbd-mirror (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:20.554 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:23:20.554 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:23:20.627 INFO:teuthology.orchestra.run.trial103.stdout:Setting up rbd-mirror (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:20.702 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:23:20.702 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:23:20.920 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-mgr (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:21.000 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-22T22:23:21.001 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-22T22:23:21.047 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-mgr (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:21.121 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-22T22:23:21.121 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /lib/systemd/system/ceph-mgr.target. 2026-02-22T22:23:21.349 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-osd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:21.433 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-22T22:23:21.434 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-22T22:23:21.487 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-osd (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:21.572 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-22T22:23:21.572 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /lib/systemd/system/ceph-osd.target. 2026-02-22T22:23:21.830 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-mgr-k8sevents (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:21.831 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-mgr-diskprediction-local (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:21.841 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-mon (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:21.914 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-22T22:23:21.914 INFO:teuthology.orchestra.run.trial190.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-22T22:23:21.947 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-mgr-k8sevents (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:21.948 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-mgr-diskprediction-local (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:21.958 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-mon (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:22.030 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-22T22:23:22.030 INFO:teuthology.orchestra.run.trial103.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /lib/systemd/system/ceph-mon.target. 2026-02-22T22:23:22.294 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-mgr-cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:22.305 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:22.306 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-mgr-dashboard (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:22.317 INFO:teuthology.orchestra.run.trial190.stdout:Setting up ceph-volume (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:22.377 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-mgr-cephadm (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:22.387 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:22.388 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-mgr-dashboard (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:22.398 INFO:teuthology.orchestra.run.trial103.stdout:Setting up ceph-volume (19.2.3-689-gf90f4eae-1jammy) ... 2026-02-22T22:23:22.432 INFO:teuthology.orchestra.run.trial190.stdout:Processing triggers for mailcap (3.70+nmu1ubuntu1) ... 2026-02-22T22:23:22.440 INFO:teuthology.orchestra.run.trial190.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-02-22T22:23:22.508 INFO:teuthology.orchestra.run.trial103.stdout:Processing triggers for mailcap (3.70+nmu1ubuntu1) ... 2026-02-22T22:23:22.515 INFO:teuthology.orchestra.run.trial103.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.13) ... 2026-02-22T22:23:22.684 INFO:teuthology.orchestra.run.trial190.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-02-22T22:23:22.755 INFO:teuthology.orchestra.run.trial103.stdout:Processing triggers for man-db (2.10.2-1) ... 2026-02-22T22:23:22.768 INFO:teuthology.orchestra.run.trial190.stdout:Processing triggers for install-info (6.8-4build1) ... 2026-02-22T22:23:22.840 INFO:teuthology.orchestra.run.trial103.stdout:Processing triggers for install-info (6.8-4build1) ... 2026-02-22T22:23:23.264 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:23.264 INFO:teuthology.orchestra.run.trial190.stdout:Running kernel seems to be up-to-date. 2026-02-22T22:23:23.264 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:23.264 INFO:teuthology.orchestra.run.trial190.stdout:The processor microcode seems to be up-to-date. 2026-02-22T22:23:23.264 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:23.264 INFO:teuthology.orchestra.run.trial190.stdout:No services need to be restarted. 2026-02-22T22:23:23.264 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:23.264 INFO:teuthology.orchestra.run.trial190.stdout:No containers need to be restarted. 2026-02-22T22:23:23.265 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:23.265 INFO:teuthology.orchestra.run.trial190.stdout:No user sessions are running outdated binaries. 2026-02-22T22:23:23.265 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:23.265 INFO:teuthology.orchestra.run.trial190.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-22T22:23:23.345 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:23.345 INFO:teuthology.orchestra.run.trial103.stdout:Running kernel seems to be up-to-date. 2026-02-22T22:23:23.345 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:23.345 INFO:teuthology.orchestra.run.trial103.stdout:The processor microcode seems to be up-to-date. 2026-02-22T22:23:23.345 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:23.345 INFO:teuthology.orchestra.run.trial103.stdout:No services need to be restarted. 2026-02-22T22:23:23.346 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:23.346 INFO:teuthology.orchestra.run.trial103.stdout:No containers need to be restarted. 2026-02-22T22:23:23.346 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:23.346 INFO:teuthology.orchestra.run.trial103.stdout:No user sessions are running outdated binaries. 2026-02-22T22:23:23.346 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:23.346 INFO:teuthology.orchestra.run.trial103.stdout:No VM guests are running outdated hypervisor (qemu) binaries on this host. 2026-02-22T22:23:24.102 INFO:teuthology.orchestra.run.trial190.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-02-22T22:23:24.109 DEBUG:teuthology.parallel:result is None 2026-02-22T22:23:24.149 INFO:teuthology.orchestra.run.trial103.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2026-02-22T22:23:24.156 DEBUG:teuthology.parallel:result is None 2026-02-22T22:23:24.156 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:23:24.319 DEBUG:teuthology.orchestra.run.trial103:> dpkg-query -W -f '${Version}' ceph 2026-02-22T22:23:24.329 INFO:teuthology.orchestra.run.trial103.stdout:19.2.3-689-gf90f4eae-1jammy 2026-02-22T22:23:24.330 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689-gf90f4eae-1jammy 2026-02-22T22:23:24.330 INFO:teuthology.task.install:The correct ceph version 19.2.3-689-gf90f4eae-1jammy is installed. 2026-02-22T22:23:24.333 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:23:24.479 DEBUG:teuthology.orchestra.run.trial190:> dpkg-query -W -f '${Version}' ceph 2026-02-22T22:23:24.489 INFO:teuthology.orchestra.run.trial190.stdout:19.2.3-689-gf90f4eae-1jammy 2026-02-22T22:23:24.490 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689-gf90f4eae-1jammy 2026-02-22T22:23:24.490 INFO:teuthology.task.install:The correct ceph version 19.2.3-689-gf90f4eae-1jammy is installed. 2026-02-22T22:23:24.492 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-22T22:23:24.492 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:24.492 DEBUG:teuthology.orchestra.run.trial103:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T22:23:24.503 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:24.503 DEBUG:teuthology.orchestra.run.trial190:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T22:23:24.541 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-22T22:23:24.541 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:24.541 DEBUG:teuthology.orchestra.run.trial103:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T22:23:24.556 DEBUG:teuthology.orchestra.run.trial103:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T22:23:24.605 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:24.605 DEBUG:teuthology.orchestra.run.trial190:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T22:23:24.617 DEBUG:teuthology.orchestra.run.trial190:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T22:23:24.668 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-22T22:23:24.669 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:24.669 DEBUG:teuthology.orchestra.run.trial103:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T22:23:24.678 DEBUG:teuthology.orchestra.run.trial103:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T22:23:24.729 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:24.729 DEBUG:teuthology.orchestra.run.trial190:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T22:23:24.740 DEBUG:teuthology.orchestra.run.trial190:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T22:23:24.788 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-22T22:23:24.789 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:24.789 DEBUG:teuthology.orchestra.run.trial103:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T22:23:24.798 DEBUG:teuthology.orchestra.run.trial103:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T22:23:24.849 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:24.850 DEBUG:teuthology.orchestra.run.trial190:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T22:23:24.860 DEBUG:teuthology.orchestra.run.trial190:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T22:23:24.908 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T22:23:25.020 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-22T22:23:25.021 DEBUG:teuthology.orchestra.run.trial103:> sudo chmod 777 /var/log/ceph 2026-02-22T22:23:25.022 DEBUG:teuthology.orchestra.run.trial190:> sudo chmod 777 /var/log/ceph 2026-02-22T22:23:25.029 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-22T22:23:25.029 DEBUG:teuthology.orchestra.run.trial103:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T22:23:25.070 DEBUG:teuthology.orchestra.run.trial190:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T22:23:25.080 INFO:tasks.ceph:Creating extra log directories... 2026-02-22T22:23:25.080 DEBUG:teuthology.orchestra.run.trial103:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T22:23:25.118 DEBUG:teuthology.orchestra.run.trial190:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T22:23:25.145 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-22T22:23:25.146 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T22:23:25.146 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-ha-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64215', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror_ha.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial103.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJStKL36rec08+9wIWCxUicGZX4AdrDb46i8HomuX0QVr2fL97guGi9kcXpCaGJ4WhA3TAw3hAIN+nvpUOYsYcs=', 'trial190.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEyAAOy4lMO91njfkj1AgusrY18FeVwgcFvwyeLtBVYRVORdeIGWSybVFvdCKSY06oL98dZ4st4HimDYzbkhVkk='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_ha.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T22:23:25.147 DEBUG:teuthology.orchestra.run.trial103:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T22:23:25.167 DEBUG:teuthology.orchestra.run.trial190:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T22:23:25.172 DEBUG:teuthology.orchestra.run.trial103:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:23:25.215 DEBUG:teuthology.orchestra.run.trial190:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:23:25.224 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:25.224 DEBUG:teuthology.orchestra.run.trial103:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T22:23:25.269 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T22:23:25.270 DEBUG:teuthology.orchestra.run.trial103:> stat /dev/vg_nvme/lv_1 2026-02-22T22:23:25.317 INFO:teuthology.orchestra.run.trial103.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T22:23:25.317 INFO:teuthology.orchestra.run.trial103.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:23:25.317 INFO:teuthology.orchestra.run.trial103.stdout:Device: 5h/5d Inode: 897 Links: 1 2026-02-22T22:23:25.318 INFO:teuthology.orchestra.run.trial103.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:23:25.318 INFO:teuthology.orchestra.run.trial103.stdout:Access: 2026-02-22 22:22:36.055129653 +0000 2026-02-22T22:23:25.318 INFO:teuthology.orchestra.run.trial103.stdout:Modify: 2026-02-22 22:22:35.887132511 +0000 2026-02-22T22:23:25.318 INFO:teuthology.orchestra.run.trial103.stdout:Change: 2026-02-22 22:22:35.887132511 +0000 2026-02-22T22:23:25.318 INFO:teuthology.orchestra.run.trial103.stdout: Birth: - 2026-02-22T22:23:25.318 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T22:23:25.368 INFO:teuthology.orchestra.run.trial103.stderr:1+0 records in 2026-02-22T22:23:25.369 INFO:teuthology.orchestra.run.trial103.stderr:1+0 records out 2026-02-22T22:23:25.369 INFO:teuthology.orchestra.run.trial103.stderr:512 bytes copied, 0.000188181 s, 2.7 MB/s 2026-02-22T22:23:25.370 DEBUG:teuthology.orchestra.run.trial103:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T22:23:25.419 DEBUG:teuthology.orchestra.run.trial103:> stat /dev/vg_nvme/lv_2 2026-02-22T22:23:25.464 INFO:teuthology.orchestra.run.trial103.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T22:23:25.464 INFO:teuthology.orchestra.run.trial103.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:23:25.464 INFO:teuthology.orchestra.run.trial103.stdout:Device: 5h/5d Inode: 921 Links: 1 2026-02-22T22:23:25.465 INFO:teuthology.orchestra.run.trial103.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:23:25.465 INFO:teuthology.orchestra.run.trial103.stdout:Access: 2026-02-22 22:22:36.395123861 +0000 2026-02-22T22:23:25.465 INFO:teuthology.orchestra.run.trial103.stdout:Modify: 2026-02-22 22:22:36.203127132 +0000 2026-02-22T22:23:25.465 INFO:teuthology.orchestra.run.trial103.stdout:Change: 2026-02-22 22:22:36.203127132 +0000 2026-02-22T22:23:25.465 INFO:teuthology.orchestra.run.trial103.stdout: Birth: - 2026-02-22T22:23:25.465 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T22:23:25.512 INFO:teuthology.orchestra.run.trial103.stderr:1+0 records in 2026-02-22T22:23:25.512 INFO:teuthology.orchestra.run.trial103.stderr:1+0 records out 2026-02-22T22:23:25.513 INFO:teuthology.orchestra.run.trial103.stderr:512 bytes copied, 0.00018256 s, 2.8 MB/s 2026-02-22T22:23:25.514 DEBUG:teuthology.orchestra.run.trial103:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T22:23:25.563 DEBUG:teuthology.orchestra.run.trial103:> stat /dev/vg_nvme/lv_3 2026-02-22T22:23:25.609 INFO:teuthology.orchestra.run.trial103.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T22:23:25.609 INFO:teuthology.orchestra.run.trial103.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:23:25.610 INFO:teuthology.orchestra.run.trial103.stdout:Device: 5h/5d Inode: 947 Links: 1 2026-02-22T22:23:25.610 INFO:teuthology.orchestra.run.trial103.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:23:25.610 INFO:teuthology.orchestra.run.trial103.stdout:Access: 2026-02-22 22:22:36.547121272 +0000 2026-02-22T22:23:25.610 INFO:teuthology.orchestra.run.trial103.stdout:Modify: 2026-02-22 22:22:36.543121341 +0000 2026-02-22T22:23:25.610 INFO:teuthology.orchestra.run.trial103.stdout:Change: 2026-02-22 22:22:36.543121341 +0000 2026-02-22T22:23:25.610 INFO:teuthology.orchestra.run.trial103.stdout: Birth: - 2026-02-22T22:23:25.610 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T22:23:25.660 INFO:teuthology.orchestra.run.trial103.stderr:1+0 records in 2026-02-22T22:23:25.661 INFO:teuthology.orchestra.run.trial103.stderr:1+0 records out 2026-02-22T22:23:25.661 INFO:teuthology.orchestra.run.trial103.stderr:512 bytes copied, 0.000175507 s, 2.9 MB/s 2026-02-22T22:23:25.662 DEBUG:teuthology.orchestra.run.trial103:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T22:23:25.711 DEBUG:teuthology.orchestra.run.trial103:> stat /dev/vg_nvme/lv_4 2026-02-22T22:23:25.757 INFO:teuthology.orchestra.run.trial103.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T22:23:25.757 INFO:teuthology.orchestra.run.trial103.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:23:25.757 INFO:teuthology.orchestra.run.trial103.stdout:Device: 5h/5d Inode: 973 Links: 1 2026-02-22T22:23:25.757 INFO:teuthology.orchestra.run.trial103.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:23:25.757 INFO:teuthology.orchestra.run.trial103.stdout:Access: 2026-02-22 22:22:37.075112277 +0000 2026-02-22T22:23:25.757 INFO:teuthology.orchestra.run.trial103.stdout:Modify: 2026-02-22 22:22:36.891115414 +0000 2026-02-22T22:23:25.758 INFO:teuthology.orchestra.run.trial103.stdout:Change: 2026-02-22 22:22:36.891115414 +0000 2026-02-22T22:23:25.758 INFO:teuthology.orchestra.run.trial103.stdout: Birth: - 2026-02-22T22:23:25.758 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T22:23:25.804 INFO:teuthology.orchestra.run.trial103.stderr:1+0 records in 2026-02-22T22:23:25.804 INFO:teuthology.orchestra.run.trial103.stderr:1+0 records out 2026-02-22T22:23:25.804 INFO:teuthology.orchestra.run.trial103.stderr:512 bytes copied, 0.000175627 s, 2.9 MB/s 2026-02-22T22:23:25.805 DEBUG:teuthology.orchestra.run.trial103:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T22:23:25.854 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:23:25.855 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial103.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:23:25.855 INFO:tasks.ceph:Generating config... 2026-02-22T22:23:25.857 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T22:23:25.857 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T22:23:25.857 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T22:23:25.857 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T22:23:25.857 INFO:tasks.ceph:[client] rbd mirror image policy type = none 2026-02-22T22:23:25.858 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:25.858 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:25.858 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:25.858 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:25.858 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:25.858 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:25.858 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:25.858 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:25.858 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:25.858 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:25.859 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:25.859 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:25.859 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:25.859 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:25.859 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T22:23:25.859 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T22:23:25.859 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T22:23:25.859 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T22:23:25.859 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T22:23:25.859 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T22:23:25.859 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T22:23:25.859 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T22:23:25.860 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T22:23:25.861 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T22:23:25.861 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T22:23:25.861 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-22T22:23:25.861 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-22T22:23:25.913 INFO:teuthology.orchestra.run.trial103.stdout:creating /etc/ceph/cluster1.keyring 2026-02-22T22:23:25.916 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-22T22:23:25.976 DEBUG:teuthology.orchestra.run.trial103:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:23:26.025 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.103')] 2026-02-22T22:23:26.026 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.103', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T22:23:26.027 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:26.027 DEBUG:teuthology.orchestra.run.trial103:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:23:26.070 DEBUG:teuthology.orchestra.run.trial103:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.103 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:monmaptool: generated fsid a7533019-ce65-46ba-9683-fc039f8c54d6 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:setting min_mon_release = quincy 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:epoch 0 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:fsid a7533019-ce65-46ba-9683-fc039f8c54d6 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:last_changed 2026-02-22T22:23:26.129123+0000 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:created 2026-02-22T22:23:26.129123+0000 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:min_mon_release 17 (quincy) 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:election_strategy: 1 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:0: [v2:10.20.193.103:3300/0,v1:10.20.193.103:6789/0] mon.a 2026-02-22T22:23:26.130 INFO:teuthology.orchestra.run.trial103.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-22T22:23:26.131 DEBUG:teuthology.orchestra.run.trial103:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:23:26.176 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID a7533019-ce65-46ba-9683-fc039f8c54d6... 2026-02-22T22:23:26.176 DEBUG:teuthology.orchestra.run.trial103:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T22:23:26.218 DEBUG:teuthology.orchestra.run.trial190:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T22:23:26.236 INFO:teuthology.orchestra.run.trial103.stdout:[global] 2026-02-22T22:23:26.236 INFO:teuthology.orchestra.run.trial103.stdout: chdir = "" 2026-02-22T22:23:26.236 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:23:26.236 INFO:teuthology.orchestra.run.trial103.stdout: auth supported = cephx 2026-02-22T22:23:26.236 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.236 INFO:teuthology.orchestra.run.trial103.stdout: filestore xattr use omap = true 2026-02-22T22:23:26.236 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon clock drift allowed = 1.000 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: osd crush chooseleaf type = 0 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: auth debug = true 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: ms die on old message = true 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: ms die on bug = true 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon pg warn max object skew = 0 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: osd pool default size = 2 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon osd allow primary affinity = true 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon osd allow pg remap = true 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on no sortbitwise = false 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on too few osds = false 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon_allow_pool_size_one = true 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: osd default data pool replay window = 5 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon allow pool delete = true 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon cluster log file level = debug 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: debug asserts on shutdown = true 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon health detail to clog = false 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon host = 10.20.193.103 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon client directed command retry = 5 2026-02-22T22:23:26.237 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on pool no app = False 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: ms inject socket failures = 5000 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: fsid = a7533019-ce65-46ba-9683-fc039f8c54d6 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout:[osd] 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd journal size = 100 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd scrub load threshold = 5.0 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd scrub max interval = 600 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd mclock skip benchmark = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd recover clone overlap = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd recovery max chunk = 1048576 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd debug shutdown = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd debug op order = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd debug verify stray on activate = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd debug trim objects = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd open classes on start = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd debug pg log writeout = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd map max advance = 10 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: journal zero on create = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: filestore apply finisher threads = 3 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: bdev debug aio = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: osd debug misdirected ops = true 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: bluefs allocator = hybrid 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: bluestore allocator = hybrid 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: bluestore block size = 96636764160 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: bluestore fsck on mount = True 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: debug bluefs = 1/20 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: debug bluestore = 1/20 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: debug ms = 1 2026-02-22T22:23:26.238 INFO:teuthology.orchestra.run.trial103.stdout: debug osd = 20 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: debug rocksdb = 4/10 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon osd full ratio = 0.9 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: osd objectstore = bluestore 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout:[mgr] 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: debug ms = 1 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: debug mgr = 20 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: debug mon = 20 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: debug auth = 20 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mgr/telemetry/nag = false 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout:[mon] 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: debug ms = 1 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: debug mon = 20 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: debug paxos = 20 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: debug auth = 20 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon data avail warn = 5 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon mgr mkfs grace = 240 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon osd reporter subtree level = osd 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon osd prime pg temp = true 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: # 1m isn't quite enough 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:26.239 INFO:teuthology.orchestra.run.trial103.stdout:[client] 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: rgw cache enabled = true 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: rgw enable ops log = true 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: rgw enable usage log = true 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: debug rbd = 20 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: debug rbd_mirror = 15 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: log to stderr = False 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: rbd default features = 125 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: rbd mirror image policy type = none 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout:[mon.a] 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.0] 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.1] 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.2] 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.3] 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.4] 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.5] 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.6] 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial190.stdout:[global] 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial190.stdout: chdir = "" 2026-02-22T22:23:26.240 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: auth supported = cephx 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: filestore xattr use omap = true 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon clock drift allowed = 1.000 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: osd crush chooseleaf type = 0 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: auth debug = true 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: ms die on old message = true 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: ms die on bug = true 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon pg warn max object skew = 0 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: osd pool default size = 2 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon osd allow primary affinity = true 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon osd allow pg remap = true 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on no sortbitwise = false 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on too few osds = false 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon_allow_pool_size_one = true 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: osd default data pool replay window = 5 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon allow pool delete = true 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon cluster log file level = debug 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: debug asserts on shutdown = true 2026-02-22T22:23:26.241 INFO:teuthology.orchestra.run.trial190.stdout: mon health detail to clog = false 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: mon host = 10.20.193.103 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: mon client directed command retry = 5 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on pool no app = False 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: ms inject socket failures = 5000 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: fsid = a7533019-ce65-46ba-9683-fc039f8c54d6 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout:[osd] 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd journal size = 100 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd scrub load threshold = 5.0 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd scrub max interval = 600 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd mclock skip benchmark = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd recover clone overlap = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd recovery max chunk = 1048576 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd debug shutdown = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd debug op order = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd debug verify stray on activate = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd debug trim objects = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd open classes on start = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd debug pg log writeout = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd map max advance = 10 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: journal zero on create = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: filestore apply finisher threads = 3 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: bdev debug aio = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: osd debug misdirected ops = true 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: bluefs allocator = hybrid 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: bluestore allocator = hybrid 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: bluestore block size = 96636764160 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: bluestore fsck on mount = True 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: debug bluefs = 1/20 2026-02-22T22:23:26.242 INFO:teuthology.orchestra.run.trial190.stdout: debug bluestore = 1/20 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug ms = 1 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug osd = 20 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug rocksdb = 4/10 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon osd full ratio = 0.9 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: osd objectstore = bluestore 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout:[mgr] 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug ms = 1 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug mgr = 20 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug mon = 20 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug auth = 20 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mgr/telemetry/nag = false 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout:[mon] 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug ms = 1 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug mon = 20 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug paxos = 20 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: debug auth = 20 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon data avail warn = 5 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon mgr mkfs grace = 240 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon osd reporter subtree level = osd 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon osd prime pg temp = true 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: # 1m isn't quite enough 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.243 INFO:teuthology.orchestra.run.trial190.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout:[client] 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: rgw cache enabled = true 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: rgw enable ops log = true 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: rgw enable usage log = true 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: debug rbd = 20 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: debug rbd_mirror = 15 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: log to stderr = False 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: rbd default features = 125 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: rbd mirror image policy type = none 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout:[mon.a] 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.0] 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.1] 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.2] 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.3] 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.4] 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.5] 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.6] 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:23:26.244 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:23:26.248 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-22T22:23:26.248 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-22T22:23:26.306 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T22:23:26.306 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:26.306 DEBUG:teuthology.orchestra.run.trial103:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-22T22:23:26.353 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:26.353 DEBUG:teuthology.orchestra.run.trial103:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-22T22:23:26.396 INFO:tasks.ceph:Sending monmap to node ubuntu@trial103.front.sepia.ceph.com 2026-02-22T22:23:26.396 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:26.396 DEBUG:teuthology.orchestra.run.trial103:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T22:23:26.396 DEBUG:teuthology.orchestra.run.trial103:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:23:26.450 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:26.450 DEBUG:teuthology.orchestra.run.trial103:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:23:26.498 INFO:tasks.ceph:Sending monmap to node ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:23:26.498 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:26.498 DEBUG:teuthology.orchestra.run.trial190:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T22:23:26.498 DEBUG:teuthology.orchestra.run.trial190:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:23:26.514 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:26.514 DEBUG:teuthology.orchestra.run.trial190:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:23:26.560 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T22:23:26.560 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T22:23:26.561 DEBUG:teuthology.orchestra.run.trial103:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T22:23:26.583 INFO:teuthology.orchestra.run.trial103.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T22:23:26.585 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T22:23:26.585 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T22:23:26.586 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-22T22:23:26.646 INFO:teuthology.orchestra.run.trial103.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-22T22:23:26.653 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T22:23:26.673 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T22:23:26.681 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T22:23:26.747 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T22:23:26.757 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T22:23:26.817 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T22:23:26.823 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T22:23:26.886 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T22:23:26.893 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T22:23:26.952 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T22:23:26.958 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T22:23:27.016 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T22:23:27.023 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T22:23:27.080 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T22:23:27.088 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T22:23:27.144 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T22:23:27.151 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T22:23:27.151 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial103.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:23:27.151 DEBUG:teuthology.orchestra.run.trial103:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-22T22:23:27.159 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:23:27.159 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-22T22:23:27.159 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial103.front.sepia.ceph.com 2026-02-22T22:23:27.159 DEBUG:teuthology.orchestra.run.trial103:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T22:23:27.211 INFO:teuthology.orchestra.run.trial103.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:23:27.211 INFO:teuthology.orchestra.run.trial103.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:23:27.211 INFO:teuthology.orchestra.run.trial103.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:23:27.211 INFO:teuthology.orchestra.run.trial103.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T22:23:27.211 INFO:teuthology.orchestra.run.trial103.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:23:27.211 INFO:teuthology.orchestra.run.trial103.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:23:27.211 INFO:teuthology.orchestra.run.trial103.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:23:27.211 INFO:teuthology.orchestra.run.trial103.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:23:27.211 INFO:teuthology.orchestra.run.trial103.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:23:27.211 INFO:teuthology.orchestra.run.trial103.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:23:27.834 INFO:teuthology.orchestra.run.trial103.stdout:Discarding blocks...Done. 2026-02-22T22:23:27.836 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial103.front.sepia.ceph.com -o noatime 2026-02-22T22:23:27.836 DEBUG:teuthology.orchestra.run.trial103:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-22T22:23:27.893 DEBUG:teuthology.orchestra.run.trial103:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-22T22:23:27.939 INFO:teuthology.orchestra.run.trial103.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T22:23:27.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:23:27.940 DEBUG:teuthology.orchestra.run.trial103:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-22T22:23:27.989 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:23:27.989 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-22T22:23:27.989 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial103.front.sepia.ceph.com 2026-02-22T22:23:27.989 DEBUG:teuthology.orchestra.run.trial103:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T22:23:28.038 INFO:teuthology.orchestra.run.trial103.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:23:28.038 INFO:teuthology.orchestra.run.trial103.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:23:28.038 INFO:teuthology.orchestra.run.trial103.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:23:28.038 INFO:teuthology.orchestra.run.trial103.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T22:23:28.038 INFO:teuthology.orchestra.run.trial103.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:23:28.038 INFO:teuthology.orchestra.run.trial103.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:23:28.038 INFO:teuthology.orchestra.run.trial103.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:23:28.038 INFO:teuthology.orchestra.run.trial103.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:23:28.038 INFO:teuthology.orchestra.run.trial103.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:23:28.038 INFO:teuthology.orchestra.run.trial103.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:23:28.643 INFO:teuthology.orchestra.run.trial103.stdout:Discarding blocks...Done. 2026-02-22T22:23:28.645 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial103.front.sepia.ceph.com -o noatime 2026-02-22T22:23:28.645 DEBUG:teuthology.orchestra.run.trial103:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-22T22:23:28.701 DEBUG:teuthology.orchestra.run.trial103:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-22T22:23:28.750 INFO:teuthology.orchestra.run.trial103.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T22:23:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:23:28.751 DEBUG:teuthology.orchestra.run.trial103:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-22T22:23:28.800 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:23:28.800 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-22T22:23:28.800 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial103.front.sepia.ceph.com 2026-02-22T22:23:28.800 DEBUG:teuthology.orchestra.run.trial103:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T22:23:28.851 INFO:teuthology.orchestra.run.trial103.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:23:28.851 INFO:teuthology.orchestra.run.trial103.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:23:28.851 INFO:teuthology.orchestra.run.trial103.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:23:28.851 INFO:teuthology.orchestra.run.trial103.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T22:23:28.851 INFO:teuthology.orchestra.run.trial103.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:23:28.851 INFO:teuthology.orchestra.run.trial103.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:23:28.851 INFO:teuthology.orchestra.run.trial103.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:23:28.851 INFO:teuthology.orchestra.run.trial103.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:23:28.851 INFO:teuthology.orchestra.run.trial103.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:23:28.851 INFO:teuthology.orchestra.run.trial103.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:23:29.417 INFO:teuthology.orchestra.run.trial103.stdout:Discarding blocks...Done. 2026-02-22T22:23:29.419 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial103.front.sepia.ceph.com -o noatime 2026-02-22T22:23:29.419 DEBUG:teuthology.orchestra.run.trial103:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-22T22:23:29.478 DEBUG:teuthology.orchestra.run.trial103:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-22T22:23:29.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:23:29.528 INFO:teuthology.orchestra.run.trial103.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T22:23:29.528 DEBUG:teuthology.orchestra.run.trial103:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:23:29.590 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:29.585+0000 7f4cf32a08c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-22T22:23:29.590 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:29.585+0000 7f4cf32a08c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-22T22:23:29.590 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:29.585+0000 7f4cf32a08c0 -1 bdev(0x55b2ee4a7800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-22T22:23:29.591 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:29.585+0000 7f4cf32a08c0 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-22T22:23:32.666 DEBUG:teuthology.orchestra.run.trial103:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-22T22:23:32.718 DEBUG:teuthology.orchestra.run.trial103:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:23:32.782 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:32.776+0000 7f4d482c88c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-22T22:23:32.783 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:32.776+0000 7f4d482c88c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-22T22:23:32.783 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:32.776+0000 7f4d482c88c0 -1 bdev(0x5576f48e7800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-22T22:23:32.784 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:32.776+0000 7f4d482c88c0 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-22T22:23:35.902 DEBUG:teuthology.orchestra.run.trial103:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-22T22:23:35.954 DEBUG:teuthology.orchestra.run.trial103:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:23:36.018 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:36.012+0000 7fbaf86968c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-22T22:23:36.018 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:36.012+0000 7fbaf86968c0 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-22T22:23:36.018 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:36.012+0000 7fbaf86968c0 -1 bdev(0x55fa8d625800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-22T22:23:36.019 INFO:teuthology.orchestra.run.trial103.stderr:2026-02-22T22:23:36.012+0000 7fbaf86968c0 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-22T22:23:39.104 DEBUG:teuthology.orchestra.run.trial103:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-22T22:23:39.154 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T22:23:39.154 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:39.155 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-22T22:23:39.206 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:39.207 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-22T22:23:39.258 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:39.259 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-22T22:23:39.310 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:39.310 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-22T22:23:39.363 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:39.363 DEBUG:teuthology.orchestra.run.trial103:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-22T22:23:39.410 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:39.410 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-22T22:23:39.415 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:39.415 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-22T22:23:39.461 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:39.461 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-22T22:23:39.508 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:39.508 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-22T22:23:39.553 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:39.553 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-22T22:23:39.600 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:39.601 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-22T22:23:39.645 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:39.645 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-22T22:23:39.690 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:23:39.690 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-22T22:23:39.736 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T22:23:39.737 DEBUG:teuthology.orchestra.run.trial103:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-22T22:23:39.745 INFO:teuthology.orchestra.run.trial103.stdout:[mgr.x] 2026-02-22T22:23:39.746 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDegZtpZgWwIhAAbXyO6DKxuQP8BmB7Ps0h9Q== 2026-02-22T22:23:39.746 INFO:teuthology.orchestra.run.trial103.stdout:[osd.0] 2026-02-22T22:23:39.746 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDhgZtpOmwiIxAA7VJYt5x+mp6A2uaCuhE/PQ== 2026-02-22T22:23:39.746 INFO:teuthology.orchestra.run.trial103.stdout:[osd.1] 2026-02-22T22:23:39.746 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDkgZtpqeeULhAA8kR2+9oSpY9Vjm9+T1YKew== 2026-02-22T22:23:39.746 INFO:teuthology.orchestra.run.trial103.stdout:[osd.2] 2026-02-22T22:23:39.746 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDogZtpOUr/ABAAJtEXG2Qu1KF2PFcFJGzzSw== 2026-02-22T22:23:39.746 INFO:teuthology.orchestra.run.trial103.stdout:[client.0] 2026-02-22T22:23:39.746 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDegZtpxwR0JhAAhsT5iePE2GXDFNaBYCUeUg== 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror] 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDegZtpt3IXKBAAeLd6y0dqxL5/hlELgIlIDA== 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.0] 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDegZtpRS9+LBAAgtVC4L92btWj0SAiket+EA== 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.1] 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDegZtpjlCjMBAA1wq5iUdSmBBjzJLWW8ukfg== 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.2] 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDegZtpcNPDNBAAeDfsVrgH68tWW6zqt80+6Q== 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.3] 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDegZtpfrW0OBAAfwpS5cE/72S5R/mRzXUddA== 2026-02-22T22:23:39.747 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.4] 2026-02-22T22:23:39.748 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDfgZtpvf/sABAAv1A9ajbXYrwrHUX+DUbbZg== 2026-02-22T22:23:39.748 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.5] 2026-02-22T22:23:39.748 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDfgZtpW5e+BBAASc0hf5UVy8G8Ydq7JhlhGQ== 2026-02-22T22:23:39.748 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.6] 2026-02-22T22:23:39.748 INFO:teuthology.orchestra.run.trial103.stdout: key = AQDfgZtpeKCQCBAAxFB8yaynqUBFBWkwLyFCXg== 2026-02-22T22:23:39.749 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T22:23:39.811 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:23:39.875 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:23:39.939 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:23:40.002 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:23:40.068 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:23:40.135 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:23:40.199 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:23:40.266 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:23:40.335 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:23:40.398 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:23:40.462 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:23:40.527 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:23:40.594 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T22:23:40.594 DEBUG:teuthology.orchestra.run.trial103:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-22T22:23:40.649 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-22T22:23:40.722 DEBUG:teuthology.orchestra.run.trial103:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-22T22:23:40.773 DEBUG:teuthology.orchestra.run.trial103:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:23:40.821 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-22T22:23:40.821 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-22T22:23:40.821 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-22T22:23:40.862 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-22T22:23:40.863 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-22T22:23:40.863 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-22T22:23:40.863 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-22T22:23:40.865 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-22T22:23:40.865 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T22:23:40.866 DEBUG:teuthology.orchestra.run.trial103:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-22T22:23:40.988 INFO:teuthology.orchestra.run.trial103.stderr:adjusted tunables profile to default 2026-02-22T22:23:41.001 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T22:23:41.001 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-22T22:23:41.001 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:41.001 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-22T22:23:41.010 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:41.010 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-22T22:23:41.066 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:23:41.066 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-22T22:23:41.116 DEBUG:teuthology.orchestra.run.trial103:> sudo ceph --cluster cluster1 osd new 08a89853-1713-4d9a-9778-de259bc9f405 0 2026-02-22T22:23:41.191 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:41.184+0000 7fea57ee9280 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T22:23:41.217 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:41.212+0000 7fea57ee9280 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T22:23:41.254 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:41.248+0000 7fea57ee9280 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T22:23:41.259 INFO:teuthology.orchestra.run.trial103.stdout:0 2026-02-22T22:23:41.269 DEBUG:teuthology.orchestra.run.trial103:> sudo ceph --cluster cluster1 osd new dae311b1-56ae-4e70-b128-073cdb3bb612 1 2026-02-22T22:23:41.341 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:41.332+0000 7fea57ee9280 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T22:23:41.413 INFO:teuthology.orchestra.run.trial103.stdout:1 2026-02-22T22:23:41.421 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:41.412+0000 7fea57ee9280 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:23:41.422 DEBUG:teuthology.orchestra.run.trial103:> sudo ceph --cluster cluster1 osd new f3ae4fde-cfef-46fb-998b-4ddceb4188d0 2 2026-02-22T22:23:41.458 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:41.452+0000 7fea57ee9280 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T22:23:41.533 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:41.524+0000 7fea57ee9280 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T22:23:41.558 INFO:teuthology.orchestra.run.trial103.stdout:2 2026-02-22T22:23:41.569 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-22T22:23:41.569 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-22T22:23:41.570 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-22T22:23:41.571 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-22T22:23:41.571 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-22T22:23:41.573 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-22T22:23:41.573 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-22T22:23:41.573 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-22T22:23:41.575 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-22T22:23:41.575 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:23:41.667 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:41.667 INFO:teuthology.orchestra.run.trial103.stdout:{"epoch":5,"fsid":"a7533019-ce65-46ba-9683-fc039f8c54d6","created":"2026-02-22T22:23:40.945517+0000","modified":"2026-02-22T22:23:41.556987+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:23:41.556987+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"08a89853-1713-4d9a-9778-de259bc9f405","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"dae311b1-56ae-4e70-b128-073cdb3bb612","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f3ae4fde-cfef-46fb-998b-4ddceb4188d0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:23:41.677 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-22T22:23:41.678 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T22:23:41.709 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:41.700+0000 7fea57ee9280 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T22:23:41.980 DEBUG:teuthology.orchestra.run.trial103:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:23:42.013 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.004+0000 7fea57ee9280 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T22:23:42.068 INFO:teuthology.misc.health.trial103.stdout: 2026-02-22T22:23:42.068 INFO:teuthology.misc.health.trial103.stdout:{"epoch":5,"fsid":"a7533019-ce65-46ba-9683-fc039f8c54d6","created":"2026-02-22T22:23:40.945517+0000","modified":"2026-02-22T22:23:41.556987+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:23:41.556987+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"08a89853-1713-4d9a-9778-de259bc9f405","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"dae311b1-56ae-4e70-b128-073cdb3bb612","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f3ae4fde-cfef-46fb-998b-4ddceb4188d0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:23:42.077 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:23:42.107 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.100+0000 7fea57ee9280 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T22:23:42.136 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.128+0000 7fea57ee9280 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T22:23:42.168 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.160+0000 7fea57ee9280 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T22:23:42.211 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.204+0000 7fea57ee9280 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T22:23:42.313 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.304+0000 7fea57ee9280 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:23:42.508 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:ERROR:root:Module 'xmltodict' is not installed. 2026-02-22T22:23:42.683 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.676+0000 7fea57ee9280 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T22:23:42.715 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.708+0000 7fea57ee9280 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T22:23:42.810 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.804+0000 7fea57ee9280 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T22:23:42.834 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.828+0000 7fea57ee9280 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T22:23:42.861 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.856+0000 7fea57ee9280 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T22:23:42.889 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:42.880+0000 7fea57ee9280 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T22:23:43.010 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T22:23:43.010 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T22:23:43.010 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr: from numpy import show_config as show_numpy_config 2026-02-22T22:23:43.017 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:43.008+0000 7fea57ee9280 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T22:23:43.044 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:43.036+0000 7fea57ee9280 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T22:23:43.083 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:43.076+0000 7fea57ee9280 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T22:23:43.134 INFO:tasks.ceph.cluster1.osd.1.trial103.stderr:2026-02-22T22:23:43.128+0000 7fd52069f8c0 -1 Falling back to public interface 2026-02-22T22:23:43.154 INFO:tasks.ceph.cluster1.osd.0.trial103.stderr:2026-02-22T22:23:43.148+0000 7fc9581178c0 -1 Falling back to public interface 2026-02-22T22:23:43.591 INFO:tasks.ceph.cluster1.osd.2.trial103.stderr:2026-02-22T22:23:43.584+0000 7f27db0d58c0 -1 Falling back to public interface 2026-02-22T22:23:44.109 INFO:tasks.ceph.cluster1.mgr.x.trial103.stderr:2026-02-22T22:23:44.100+0000 7fe976639640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T22:23:46.937 INFO:tasks.ceph.cluster1.osd.1.trial103.stderr:2026-02-22T22:23:46.928+0000 7fd52069f8c0 -1 osd.1 0 log_to_monitors true 2026-02-22T22:23:47.516 INFO:tasks.ceph.cluster1.osd.0.trial103.stderr:2026-02-22T22:23:47.508+0000 7fc9581178c0 -1 osd.0 0 log_to_monitors true 2026-02-22T22:23:47.690 INFO:tasks.ceph.cluster1.osd.2.trial103.stderr:2026-02-22T22:23:47.684+0000 7f27db0d58c0 -1 osd.2 0 log_to_monitors true 2026-02-22T22:23:48.112 INFO:tasks.ceph.cluster1.osd.1.trial103.stderr:2026-02-22T22:23:48.104+0000 7fd51c648640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T22:23:48.134 INFO:tasks.ceph.cluster1.osd.1.trial103.stderr:2026-02-22T22:23:48.128+0000 7fd51447b640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:23:48.379 DEBUG:teuthology.orchestra.run.trial103:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:23:48.517 INFO:teuthology.misc.health.trial103.stdout: 2026-02-22T22:23:48.518 INFO:teuthology.misc.health.trial103.stdout:{"epoch":7,"fsid":"a7533019-ce65-46ba-9683-fc039f8c54d6","created":"2026-02-22T22:23:40.945517+0000","modified":"2026-02-22T22:23:48.105863+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:23:41.556987+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"08a89853-1713-4d9a-9778-de259bc9f405","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"dae311b1-56ae-4e70-b128-073cdb3bb612","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f3ae4fde-cfef-46fb-998b-4ddceb4188d0","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:23:48.527 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:23:49.114 INFO:tasks.ceph.cluster1.osd.0.trial103.stderr:2026-02-22T22:23:49.108+0000 7fc9540c0640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T22:23:49.115 INFO:tasks.ceph.cluster1.osd.2.trial103.stderr:2026-02-22T22:23:49.108+0000 7f27d707e640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T22:23:49.134 INFO:tasks.ceph.cluster1.osd.0.trial103.stderr:2026-02-22T22:23:49.128+0000 7fc94bef3640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:23:49.146 INFO:tasks.ceph.cluster1.osd.2.trial103.stderr:2026-02-22T22:23:49.140+0000 7f27ceeb1640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:23:54.829 DEBUG:teuthology.orchestra.run.trial103:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:23:54.974 INFO:teuthology.misc.health.trial103.stdout: 2026-02-22T22:23:54.974 INFO:teuthology.misc.health.trial103.stdout:{"epoch":12,"fsid":"a7533019-ce65-46ba-9683-fc039f8c54d6","created":"2026-02-22T22:23:40.945517+0000","modified":"2026-02-22T22:23:54.120568+0000","last_up_change":"2026-02-22T22:23:50.110835+0000","last_in_change":"2026-02-22T22:23:41.556987+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:23:51.138178+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"08a89853-1713-4d9a-9778-de259bc9f405","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6810","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6811","nonce":3539595539}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6812","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6813","nonce":3539595539}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6816","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6817","nonce":3539595539}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6814","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6815","nonce":3539595539}]},"public_addr":"10.20.193.103:6811/3539595539","cluster_addr":"10.20.193.103:6813/3539595539","heartbeat_back_addr":"10.20.193.103:6817/3539595539","heartbeat_front_addr":"10.20.193.103:6815/3539595539","state":["exists","up"]},{"osd":1,"uuid":"dae311b1-56ae-4e70-b128-073cdb3bb612","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6802","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6803","nonce":3310548690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6804","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6805","nonce":3310548690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6808","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6809","nonce":3310548690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6806","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6807","nonce":3310548690}]},"public_addr":"10.20.193.103:6803/3310548690","cluster_addr":"10.20.193.103:6805/3310548690","heartbeat_back_addr":"10.20.193.103:6809/3310548690","heartbeat_front_addr":"10.20.193.103:6807/3310548690","state":["exists","up"]},{"osd":2,"uuid":"f3ae4fde-cfef-46fb-998b-4ddceb4188d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6818","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6819","nonce":1670334570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6820","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6821","nonce":1670334570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6824","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6825","nonce":1670334570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6822","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6823","nonce":1670334570}]},"public_addr":"10.20.193.103:6819/1670334570","cluster_addr":"10.20.193.103:6821/1670334570","heartbeat_back_addr":"10.20.193.103:6825/1670334570","heartbeat_front_addr":"10.20.193.103:6823/1670334570","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:23:54.983 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T22:23:54.983 INFO:tasks.ceph:Creating RBD pool 2026-02-22T22:23:54.983 DEBUG:teuthology.orchestra.run.trial103:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-22T22:23:56.130 INFO:teuthology.orchestra.run.trial103.stderr:pool 'rbd' created 2026-02-22T22:23:56.143 DEBUG:teuthology.orchestra.run.trial103:> rbd --cluster cluster1 pool init rbd 2026-02-22T22:23:59.145 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-22T22:23:59.146 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-22T22:23:59.146 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-22T22:23:59.146 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-22T22:23:59.147 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T22:23:59.324 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:59.333 INFO:teuthology.orchestra.run.trial103.stdout:{"epoch":5,"flags":0,"active_gid":4101,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6800","nonce":3011327028},{"type":"v1","addr":"10.20.193.103:6801","nonce":3011327028}]},"active_addr":"10.20.193.103:6801/3011327028","active_change":"2026-02-22T22:23:43.101167+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.103:0","nonce":3191730856}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.103:0","nonce":1782880954}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.103:0","nonce":3371057793}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.103:0","nonce":1969030002}]}]} 2026-02-22T22:23:59.337 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-22T22:23:59.337 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-22T22:23:59.337 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:23:59.481 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:59.482 INFO:teuthology.orchestra.run.trial103.stdout:{"epoch":16,"fsid":"a7533019-ce65-46ba-9683-fc039f8c54d6","created":"2026-02-22T22:23:40.945517+0000","modified":"2026-02-22T22:23:59.133730+0000","last_up_change":"2026-02-22T22:23:50.110835+0000","last_in_change":"2026-02-22T22:23:41.556987+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:23:51.138178+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:23:55.134505+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"08a89853-1713-4d9a-9778-de259bc9f405","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6810","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6811","nonce":3539595539}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6812","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6813","nonce":3539595539}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6816","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6817","nonce":3539595539}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6814","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6815","nonce":3539595539}]},"public_addr":"10.20.193.103:6811/3539595539","cluster_addr":"10.20.193.103:6813/3539595539","heartbeat_back_addr":"10.20.193.103:6817/3539595539","heartbeat_front_addr":"10.20.193.103:6815/3539595539","state":["exists","up"]},{"osd":1,"uuid":"dae311b1-56ae-4e70-b128-073cdb3bb612","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6802","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6803","nonce":3310548690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6804","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6805","nonce":3310548690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6808","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6809","nonce":3310548690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6806","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6807","nonce":3310548690}]},"public_addr":"10.20.193.103:6803/3310548690","cluster_addr":"10.20.193.103:6805/3310548690","heartbeat_back_addr":"10.20.193.103:6809/3310548690","heartbeat_front_addr":"10.20.193.103:6807/3310548690","state":["exists","up"]},{"osd":2,"uuid":"f3ae4fde-cfef-46fb-998b-4ddceb4188d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6818","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6819","nonce":1670334570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6820","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6821","nonce":1670334570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6824","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6825","nonce":1670334570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6822","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6823","nonce":1670334570}]},"public_addr":"10.20.193.103:6819/1670334570","cluster_addr":"10.20.193.103:6821/1670334570","heartbeat_back_addr":"10.20.193.103:6825/1670334570","heartbeat_front_addr":"10.20.193.103:6823/1670334570","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:23:59.492 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-22T22:23:59.492 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:23:59.635 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:23:59.635 INFO:teuthology.orchestra.run.trial103.stdout:{"epoch":16,"fsid":"a7533019-ce65-46ba-9683-fc039f8c54d6","created":"2026-02-22T22:23:40.945517+0000","modified":"2026-02-22T22:23:59.133730+0000","last_up_change":"2026-02-22T22:23:50.110835+0000","last_in_change":"2026-02-22T22:23:41.556987+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:23:51.138178+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:23:55.134505+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"08a89853-1713-4d9a-9778-de259bc9f405","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6810","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6811","nonce":3539595539}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6812","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6813","nonce":3539595539}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6816","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6817","nonce":3539595539}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6814","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6815","nonce":3539595539}]},"public_addr":"10.20.193.103:6811/3539595539","cluster_addr":"10.20.193.103:6813/3539595539","heartbeat_back_addr":"10.20.193.103:6817/3539595539","heartbeat_front_addr":"10.20.193.103:6815/3539595539","state":["exists","up"]},{"osd":1,"uuid":"dae311b1-56ae-4e70-b128-073cdb3bb612","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6802","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6803","nonce":3310548690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6804","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6805","nonce":3310548690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6808","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6809","nonce":3310548690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6806","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6807","nonce":3310548690}]},"public_addr":"10.20.193.103:6803/3310548690","cluster_addr":"10.20.193.103:6805/3310548690","heartbeat_back_addr":"10.20.193.103:6809/3310548690","heartbeat_front_addr":"10.20.193.103:6807/3310548690","state":["exists","up"]},{"osd":2,"uuid":"f3ae4fde-cfef-46fb-998b-4ddceb4188d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6818","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6819","nonce":1670334570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6820","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6821","nonce":1670334570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6824","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6825","nonce":1670334570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6822","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6823","nonce":1670334570}]},"public_addr":"10.20.193.103:6819/1670334570","cluster_addr":"10.20.193.103:6821/1670334570","heartbeat_back_addr":"10.20.193.103:6825/1670334570","heartbeat_front_addr":"10.20.193.103:6823/1670334570","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:23:59.648 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-22T22:23:59.648 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-22T22:23:59.649 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-22T22:23:59.732 INFO:teuthology.orchestra.run.trial103.stdout:38654705668 2026-02-22T22:23:59.732 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:23:59.733 INFO:teuthology.orchestra.run.trial103.stdout:38654705668 2026-02-22T22:23:59.734 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:23:59.737 INFO:teuthology.orchestra.run.trial103.stdout:34359738373 2026-02-22T22:23:59.738 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:23:59.923 INFO:teuthology.orchestra.run.trial103.stdout:38654705667 2026-02-22T22:23:59.928 INFO:teuthology.orchestra.run.trial103.stdout:38654705667 2026-02-22T22:23:59.934 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705667 for osd.0 2026-02-22T22:23:59.940 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705667 for osd.2 2026-02-22T22:23:59.988 INFO:teuthology.orchestra.run.trial103.stdout:34359738372 2026-02-22T22:24:00.000 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738372 for osd.1 2026-02-22T22:24:00.935 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:24:00.942 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:24:01.000 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:24:01.084 INFO:teuthology.orchestra.run.trial103.stdout:38654705667 2026-02-22T22:24:01.097 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705667 for osd.0 2026-02-22T22:24:01.123 INFO:teuthology.orchestra.run.trial103.stdout:38654705667 2026-02-22T22:24:01.135 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705667 for osd.2 2026-02-22T22:24:01.168 INFO:teuthology.orchestra.run.trial103.stdout:34359738374 2026-02-22T22:24:01.179 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738373 got 34359738374 for osd.1 2026-02-22T22:24:01.180 DEBUG:teuthology.parallel:result is None 2026-02-22T22:24:02.098 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:24:02.136 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:24:02.247 INFO:teuthology.orchestra.run.trial103.stdout:38654705668 2026-02-22T22:24:02.258 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705668 for osd.0 2026-02-22T22:24:02.258 DEBUG:teuthology.parallel:result is None 2026-02-22T22:24:02.288 INFO:teuthology.orchestra.run.trial103.stdout:38654705668 2026-02-22T22:24:02.299 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705668 for osd.2 2026-02-22T22:24:02.300 DEBUG:teuthology.parallel:result is None 2026-02-22T22:24:02.300 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T22:24:02.300 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:24:02.486 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:02.487 INFO:teuthology.orchestra.run.trial103.stderr:dumped all 2026-02-22T22:24:02.497 INFO:teuthology.orchestra.run.trial103.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T22:24:01.105622+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82476,"kb_used_data":1568,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033044,"statfs":{"total":289910292480,"available":289825837056,"internally_reserved":0,"allocated":1605632,"data_stored":1298909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":82767187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.976336"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.140328+0000","last_change":"2026-02-22T22:23:59.140328+0000","last_active":"2026-02-22T22:23:59.140328+0000","last_peered":"2026-02-22T22:23:59.140328+0000","last_clean":"2026-02-22T22:23:59.140328+0000","last_became_active":"2026-02-22T22:23:57.137067+0000","last_became_peered":"2026-02-22T22:23:57.137067+0000","last_unstale":"2026-02-22T22:23:59.140328+0000","last_undegraded":"2026-02-22T22:23:59.140328+0000","last_fullsized":"2026-02-22T22:23:59.140328+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:01:31.446177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061824999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.140089+0000","last_change":"2026-02-22T22:23:59.140089+0000","last_active":"2026-02-22T22:23:59.140089+0000","last_peered":"2026-02-22T22:23:59.140089+0000","last_clean":"2026-02-22T22:23:59.140089+0000","last_became_active":"2026-02-22T22:23:57.137050+0000","last_became_peered":"2026-02-22T22:23:57.137050+0000","last_unstale":"2026-02-22T22:23:59.140089+0000","last_undegraded":"2026-02-22T22:23:59.140089+0000","last_fullsized":"2026-02-22T22:23:59.140089+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:03:18.154802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041303899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.140496+0000","last_change":"2026-02-22T22:23:59.140496+0000","last_active":"2026-02-22T22:23:59.140496+0000","last_peered":"2026-02-22T22:23:59.140496+0000","last_clean":"2026-02-22T22:23:59.140496+0000","last_became_active":"2026-02-22T22:23:57.136991+0000","last_became_peered":"2026-02-22T22:23:57.136991+0000","last_unstale":"2026-02-22T22:23:59.140496+0000","last_undegraded":"2026-02-22T22:23:59.140496+0000","last_fullsized":"2026-02-22T22:23:59.140496+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:14:39.011925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078158500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.140094+0000","last_change":"2026-02-22T22:23:59.140094+0000","last_active":"2026-02-22T22:23:59.140094+0000","last_peered":"2026-02-22T22:23:59.140094+0000","last_clean":"2026-02-22T22:23:59.140094+0000","last_became_active":"2026-02-22T22:23:57.136930+0000","last_became_peered":"2026-02-22T22:23:57.136930+0000","last_unstale":"2026-02-22T22:23:59.140094+0000","last_undegraded":"2026-02-22T22:23:59.140094+0000","last_fullsized":"2026-02-22T22:23:59.140094+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:49:52.109397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040949200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.139101+0000","last_change":"2026-02-22T22:23:59.138515+0000","last_active":"2026-02-22T22:23:59.139101+0000","last_peered":"2026-02-22T22:23:59.139101+0000","last_clean":"2026-02-22T22:23:59.139101+0000","last_became_active":"2026-02-22T22:23:57.136352+0000","last_became_peered":"2026-02-22T22:23:57.136352+0000","last_unstale":"2026-02-22T22:23:59.139101+0000","last_undegraded":"2026-02-22T22:23:59.139101+0000","last_fullsized":"2026-02-22T22:23:59.139101+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:56.872656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061167800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.136866+0000","last_change":"2026-02-22T22:23:59.136866+0000","last_active":"2026-02-22T22:23:59.136866+0000","last_peered":"2026-02-22T22:23:59.136866+0000","last_clean":"2026-02-22T22:23:59.136866+0000","last_became_active":"2026-02-22T22:23:57.136381+0000","last_became_peered":"2026-02-22T22:23:57.136381+0000","last_unstale":"2026-02-22T22:23:59.136866+0000","last_undegraded":"2026-02-22T22:23:59.136866+0000","last_fullsized":"2026-02-22T22:23:59.136866+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:47:13.328371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023819299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.136809+0000","last_change":"2026-02-22T22:23:59.136809+0000","last_active":"2026-02-22T22:23:59.136809+0000","last_peered":"2026-02-22T22:23:59.136809+0000","last_clean":"2026-02-22T22:23:59.136809+0000","last_became_active":"2026-02-22T22:23:57.136128+0000","last_became_peered":"2026-02-22T22:23:57.136128+0000","last_unstale":"2026-02-22T22:23:59.136809+0000","last_undegraded":"2026-02-22T22:23:59.136809+0000","last_fullsized":"2026-02-22T22:23:59.136809+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:33.645236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000213277,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.140296+0000","last_change":"2026-02-22T22:23:59.140296+0000","last_active":"2026-02-22T22:23:59.140296+0000","last_peered":"2026-02-22T22:23:59.140296+0000","last_clean":"2026-02-22T22:23:59.140296+0000","last_became_active":"2026-02-22T22:23:57.136046+0000","last_became_peered":"2026-02-22T22:23:57.136046+0000","last_unstale":"2026-02-22T22:23:59.140296+0000","last_undegraded":"2026-02-22T22:23:59.140296+0000","last_fullsized":"2026-02-22T22:23:59.140296+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:27:57.025803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000595328,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.139615+0000","last_change":"2026-02-22T22:23:53.494316+0000","last_active":"2026-02-22T22:23:59.139615+0000","last_peered":"2026-02-22T22:23:59.139615+0000","last_clean":"2026-02-22T22:23:59.139615+0000","last_became_active":"2026-02-22T22:23:53.493528+0000","last_became_peered":"2026-02-22T22:23:53.493528+0000","last_unstale":"2026-02-22T22:23:59.139615+0000","last_undegraded":"2026-02-22T22:23:59.139615+0000","last_fullsized":"2026-02-22T22:23:59.139615+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_clean_scrub_stamp":"2026-02-22T22:23:52.116710+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:04:37.859993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27096,"kb_used_data":128,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344744,"statfs":{"total":96636764160,"available":96609017856,"internally_reserved":0,"allocated":131072,"data_stored":37005,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705668,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":630952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27589063},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738374,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27692,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344148,"statfs":{"total":96636764160,"available":96608407552,"internally_reserved":0,"allocated":737280,"data_stored":630952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:24:02.499 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:24:02.652 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:02.653 INFO:teuthology.orchestra.run.trial103.stderr:dumped all 2026-02-22T22:24:02.663 INFO:teuthology.orchestra.run.trial103.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T22:24:01.105622+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82476,"kb_used_data":1568,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033044,"statfs":{"total":289910292480,"available":289825837056,"internally_reserved":0,"allocated":1605632,"data_stored":1298909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":82767187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.976336"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.140328+0000","last_change":"2026-02-22T22:23:59.140328+0000","last_active":"2026-02-22T22:23:59.140328+0000","last_peered":"2026-02-22T22:23:59.140328+0000","last_clean":"2026-02-22T22:23:59.140328+0000","last_became_active":"2026-02-22T22:23:57.137067+0000","last_became_peered":"2026-02-22T22:23:57.137067+0000","last_unstale":"2026-02-22T22:23:59.140328+0000","last_undegraded":"2026-02-22T22:23:59.140328+0000","last_fullsized":"2026-02-22T22:23:59.140328+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:01:31.446177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061824999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.140089+0000","last_change":"2026-02-22T22:23:59.140089+0000","last_active":"2026-02-22T22:23:59.140089+0000","last_peered":"2026-02-22T22:23:59.140089+0000","last_clean":"2026-02-22T22:23:59.140089+0000","last_became_active":"2026-02-22T22:23:57.137050+0000","last_became_peered":"2026-02-22T22:23:57.137050+0000","last_unstale":"2026-02-22T22:23:59.140089+0000","last_undegraded":"2026-02-22T22:23:59.140089+0000","last_fullsized":"2026-02-22T22:23:59.140089+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:03:18.154802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041303899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.140496+0000","last_change":"2026-02-22T22:23:59.140496+0000","last_active":"2026-02-22T22:23:59.140496+0000","last_peered":"2026-02-22T22:23:59.140496+0000","last_clean":"2026-02-22T22:23:59.140496+0000","last_became_active":"2026-02-22T22:23:57.136991+0000","last_became_peered":"2026-02-22T22:23:57.136991+0000","last_unstale":"2026-02-22T22:23:59.140496+0000","last_undegraded":"2026-02-22T22:23:59.140496+0000","last_fullsized":"2026-02-22T22:23:59.140496+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:14:39.011925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078158500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.140094+0000","last_change":"2026-02-22T22:23:59.140094+0000","last_active":"2026-02-22T22:23:59.140094+0000","last_peered":"2026-02-22T22:23:59.140094+0000","last_clean":"2026-02-22T22:23:59.140094+0000","last_became_active":"2026-02-22T22:23:57.136930+0000","last_became_peered":"2026-02-22T22:23:57.136930+0000","last_unstale":"2026-02-22T22:23:59.140094+0000","last_undegraded":"2026-02-22T22:23:59.140094+0000","last_fullsized":"2026-02-22T22:23:59.140094+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:49:52.109397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040949200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.139101+0000","last_change":"2026-02-22T22:23:59.138515+0000","last_active":"2026-02-22T22:23:59.139101+0000","last_peered":"2026-02-22T22:23:59.139101+0000","last_clean":"2026-02-22T22:23:59.139101+0000","last_became_active":"2026-02-22T22:23:57.136352+0000","last_became_peered":"2026-02-22T22:23:57.136352+0000","last_unstale":"2026-02-22T22:23:59.139101+0000","last_undegraded":"2026-02-22T22:23:59.139101+0000","last_fullsized":"2026-02-22T22:23:59.139101+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:56.872656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061167800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.136866+0000","last_change":"2026-02-22T22:23:59.136866+0000","last_active":"2026-02-22T22:23:59.136866+0000","last_peered":"2026-02-22T22:23:59.136866+0000","last_clean":"2026-02-22T22:23:59.136866+0000","last_became_active":"2026-02-22T22:23:57.136381+0000","last_became_peered":"2026-02-22T22:23:57.136381+0000","last_unstale":"2026-02-22T22:23:59.136866+0000","last_undegraded":"2026-02-22T22:23:59.136866+0000","last_fullsized":"2026-02-22T22:23:59.136866+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:47:13.328371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023819299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.136809+0000","last_change":"2026-02-22T22:23:59.136809+0000","last_active":"2026-02-22T22:23:59.136809+0000","last_peered":"2026-02-22T22:23:59.136809+0000","last_clean":"2026-02-22T22:23:59.136809+0000","last_became_active":"2026-02-22T22:23:57.136128+0000","last_became_peered":"2026-02-22T22:23:57.136128+0000","last_unstale":"2026-02-22T22:23:59.136809+0000","last_undegraded":"2026-02-22T22:23:59.136809+0000","last_fullsized":"2026-02-22T22:23:59.136809+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:33.645236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000213277,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.140296+0000","last_change":"2026-02-22T22:23:59.140296+0000","last_active":"2026-02-22T22:23:59.140296+0000","last_peered":"2026-02-22T22:23:59.140296+0000","last_clean":"2026-02-22T22:23:59.140296+0000","last_became_active":"2026-02-22T22:23:57.136046+0000","last_became_peered":"2026-02-22T22:23:57.136046+0000","last_unstale":"2026-02-22T22:23:59.140296+0000","last_undegraded":"2026-02-22T22:23:59.140296+0000","last_fullsized":"2026-02-22T22:23:59.140296+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:27:57.025803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000595328,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:23:59.139615+0000","last_change":"2026-02-22T22:23:53.494316+0000","last_active":"2026-02-22T22:23:59.139615+0000","last_peered":"2026-02-22T22:23:59.139615+0000","last_clean":"2026-02-22T22:23:59.139615+0000","last_became_active":"2026-02-22T22:23:53.493528+0000","last_became_peered":"2026-02-22T22:23:53.493528+0000","last_unstale":"2026-02-22T22:23:59.139615+0000","last_undegraded":"2026-02-22T22:23:59.139615+0000","last_fullsized":"2026-02-22T22:23:59.139615+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_clean_scrub_stamp":"2026-02-22T22:23:52.116710+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:04:37.859993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27096,"kb_used_data":128,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344744,"statfs":{"total":96636764160,"available":96609017856,"internally_reserved":0,"allocated":131072,"data_stored":37005,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705668,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":630952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27589063},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738374,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27692,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344148,"statfs":{"total":96636764160,"available":96608407552,"internally_reserved":0,"allocated":737280,"data_stored":630952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:24:02.664 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T22:24:02.664 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-22T22:24:02.664 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-22T22:24:02.665 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-22T22:24:02.826 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:02.826 INFO:teuthology.orchestra.run.trial103.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T22:24:02.837 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-22T22:24:02.837 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T22:24:02.847 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-22T22:24:02.848 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T22:24:02.848 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-ha-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64215', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'ktype': 'distro', 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror_ha.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'ubuntu', 'os_version': '22.04', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial103.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJStKL36rec08+9wIWCxUicGZX4AdrDb46i8HomuX0QVr2fL97guGi9kcXpCaGJ4WhA3TAw3hAIN+nvpUOYsYcs=', 'trial190.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEyAAOy4lMO91njfkj1AgusrY18FeVwgcFvwyeLtBVYRVORdeIGWSybVFvdCKSY06oL98dZ4st4HimDYzbkhVkk='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_ha.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T22:24:02.849 DEBUG:teuthology.orchestra.run.trial103:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T22:24:02.851 DEBUG:teuthology.orchestra.run.trial190:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T22:24:02.857 DEBUG:teuthology.orchestra.run.trial103:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:24:02.899 DEBUG:teuthology.orchestra.run.trial190:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:24:02.909 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:02.909 DEBUG:teuthology.orchestra.run.trial190:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T22:24:02.957 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T22:24:02.957 DEBUG:teuthology.orchestra.run.trial190:> stat /dev/vg_nvme/lv_1 2026-02-22T22:24:03.005 INFO:teuthology.orchestra.run.trial190.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T22:24:03.005 INFO:teuthology.orchestra.run.trial190.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:24:03.005 INFO:teuthology.orchestra.run.trial190.stdout:Device: 5h/5d Inode: 902 Links: 1 2026-02-22T22:24:03.005 INFO:teuthology.orchestra.run.trial190.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:24:03.005 INFO:teuthology.orchestra.run.trial190.stdout:Access: 2026-02-22 22:22:36.313718874 +0000 2026-02-22T22:24:03.005 INFO:teuthology.orchestra.run.trial190.stdout:Modify: 2026-02-22 22:22:36.129722008 +0000 2026-02-22T22:24:03.005 INFO:teuthology.orchestra.run.trial190.stdout:Change: 2026-02-22 22:22:36.129722008 +0000 2026-02-22T22:24:03.006 INFO:teuthology.orchestra.run.trial190.stdout: Birth: - 2026-02-22T22:24:03.006 DEBUG:teuthology.orchestra.run.trial190:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T22:24:03.056 INFO:teuthology.orchestra.run.trial190.stderr:1+0 records in 2026-02-22T22:24:03.056 INFO:teuthology.orchestra.run.trial190.stderr:1+0 records out 2026-02-22T22:24:03.056 INFO:teuthology.orchestra.run.trial190.stderr:512 bytes copied, 0.000186858 s, 2.7 MB/s 2026-02-22T22:24:03.057 DEBUG:teuthology.orchestra.run.trial190:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T22:24:03.105 DEBUG:teuthology.orchestra.run.trial190:> stat /dev/vg_nvme/lv_2 2026-02-22T22:24:03.152 INFO:teuthology.orchestra.run.trial190.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T22:24:03.152 INFO:teuthology.orchestra.run.trial190.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:24:03.153 INFO:teuthology.orchestra.run.trial190.stdout:Device: 5h/5d Inode: 938 Links: 1 2026-02-22T22:24:03.153 INFO:teuthology.orchestra.run.trial190.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:24:03.153 INFO:teuthology.orchestra.run.trial190.stdout:Access: 2026-02-22 22:22:36.477716081 +0000 2026-02-22T22:24:03.153 INFO:teuthology.orchestra.run.trial190.stdout:Modify: 2026-02-22 22:22:36.473716147 +0000 2026-02-22T22:24:03.153 INFO:teuthology.orchestra.run.trial190.stdout:Change: 2026-02-22 22:22:36.473716147 +0000 2026-02-22T22:24:03.153 INFO:teuthology.orchestra.run.trial190.stdout: Birth: - 2026-02-22T22:24:03.153 DEBUG:teuthology.orchestra.run.trial190:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T22:24:03.203 INFO:teuthology.orchestra.run.trial190.stderr:1+0 records in 2026-02-22T22:24:03.203 INFO:teuthology.orchestra.run.trial190.stderr:1+0 records out 2026-02-22T22:24:03.203 INFO:teuthology.orchestra.run.trial190.stderr:512 bytes copied, 0.000166751 s, 3.1 MB/s 2026-02-22T22:24:03.204 DEBUG:teuthology.orchestra.run.trial190:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T22:24:03.250 DEBUG:teuthology.orchestra.run.trial190:> stat /dev/vg_nvme/lv_3 2026-02-22T22:24:03.296 INFO:teuthology.orchestra.run.trial190.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T22:24:03.296 INFO:teuthology.orchestra.run.trial190.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:24:03.296 INFO:teuthology.orchestra.run.trial190.stdout:Device: 5h/5d Inode: 953 Links: 1 2026-02-22T22:24:03.297 INFO:teuthology.orchestra.run.trial190.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:24:03.297 INFO:teuthology.orchestra.run.trial190.stdout:Access: 2026-02-22 22:22:36.817710290 +0000 2026-02-22T22:24:03.297 INFO:teuthology.orchestra.run.trial190.stdout:Modify: 2026-02-22 22:22:36.813710358 +0000 2026-02-22T22:24:03.297 INFO:teuthology.orchestra.run.trial190.stdout:Change: 2026-02-22 22:22:36.813710358 +0000 2026-02-22T22:24:03.297 INFO:teuthology.orchestra.run.trial190.stdout: Birth: - 2026-02-22T22:24:03.297 DEBUG:teuthology.orchestra.run.trial190:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T22:24:03.344 INFO:teuthology.orchestra.run.trial190.stderr:1+0 records in 2026-02-22T22:24:03.344 INFO:teuthology.orchestra.run.trial190.stderr:1+0 records out 2026-02-22T22:24:03.344 INFO:teuthology.orchestra.run.trial190.stderr:512 bytes copied, 0.000175106 s, 2.9 MB/s 2026-02-22T22:24:03.346 DEBUG:teuthology.orchestra.run.trial190:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T22:24:03.394 DEBUG:teuthology.orchestra.run.trial190:> stat /dev/vg_nvme/lv_4 2026-02-22T22:24:03.441 INFO:teuthology.orchestra.run.trial190.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T22:24:03.441 INFO:teuthology.orchestra.run.trial190.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:24:03.441 INFO:teuthology.orchestra.run.trial190.stdout:Device: 5h/5d Inode: 975 Links: 1 2026-02-22T22:24:03.441 INFO:teuthology.orchestra.run.trial190.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:24:03.441 INFO:teuthology.orchestra.run.trial190.stdout:Access: 2026-02-22 22:22:37.337701431 +0000 2026-02-22T22:24:03.441 INFO:teuthology.orchestra.run.trial190.stdout:Modify: 2026-02-22 22:22:37.157704499 +0000 2026-02-22T22:24:03.441 INFO:teuthology.orchestra.run.trial190.stdout:Change: 2026-02-22 22:22:37.157704499 +0000 2026-02-22T22:24:03.441 INFO:teuthology.orchestra.run.trial190.stdout: Birth: - 2026-02-22T22:24:03.442 DEBUG:teuthology.orchestra.run.trial190:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T22:24:03.492 INFO:teuthology.orchestra.run.trial190.stderr:1+0 records in 2026-02-22T22:24:03.492 INFO:teuthology.orchestra.run.trial190.stderr:1+0 records out 2026-02-22T22:24:03.492 INFO:teuthology.orchestra.run.trial190.stderr:512 bytes copied, 0.000170217 s, 3.0 MB/s 2026-02-22T22:24:03.493 DEBUG:teuthology.orchestra.run.trial190:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T22:24:03.541 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:24:03.541 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial190.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:24:03.541 INFO:tasks.ceph:Generating config... 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client] rbd mirror image policy type = none 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.542 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.542 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T22:24:03.543 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T22:24:03.543 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T22:24:03.543 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T22:24:03.543 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T22:24:03.543 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T22:24:03.543 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T22:24:03.543 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] bluefs allocator = hybrid 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] bluestore allocator = hybrid 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T22:24:03.543 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T22:24:03.543 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-22T22:24:03.543 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-22T22:24:03.601 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster2.keyring 2026-02-22T22:24:03.603 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-22T22:24:03.662 DEBUG:teuthology.orchestra.run.trial190:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:24:03.713 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.190')] 2026-02-22T22:24:03.713 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.190', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bluefs allocator': 'hybrid', 'bluestore allocator': 'hybrid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T22:24:03.715 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:03.715 DEBUG:teuthology.orchestra.run.trial190:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:24:03.757 DEBUG:teuthology.orchestra.run.trial190:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.190 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:24:03.818 INFO:teuthology.orchestra.run.trial190.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:24:03.818 INFO:teuthology.orchestra.run.trial190.stdout:monmaptool: generated fsid 56258423-fe54-446e-ae04-77688092fba9 2026-02-22T22:24:03.818 INFO:teuthology.orchestra.run.trial190.stdout:setting min_mon_release = quincy 2026-02-22T22:24:03.818 INFO:teuthology.orchestra.run.trial190.stdout:epoch 0 2026-02-22T22:24:03.818 INFO:teuthology.orchestra.run.trial190.stdout:fsid 56258423-fe54-446e-ae04-77688092fba9 2026-02-22T22:24:03.818 INFO:teuthology.orchestra.run.trial190.stdout:last_changed 2026-02-22T22:24:03.816339+0000 2026-02-22T22:24:03.819 INFO:teuthology.orchestra.run.trial190.stdout:created 2026-02-22T22:24:03.816339+0000 2026-02-22T22:24:03.819 INFO:teuthology.orchestra.run.trial190.stdout:min_mon_release 17 (quincy) 2026-02-22T22:24:03.819 INFO:teuthology.orchestra.run.trial190.stdout:election_strategy: 1 2026-02-22T22:24:03.819 INFO:teuthology.orchestra.run.trial190.stdout:0: [v2:10.20.193.190:3300/0,v1:10.20.193.190:6789/0] mon.a 2026-02-22T22:24:03.819 INFO:teuthology.orchestra.run.trial190.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-22T22:24:03.820 DEBUG:teuthology.orchestra.run.trial190:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:24:03.865 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 56258423-fe54-446e-ae04-77688092fba9... 2026-02-22T22:24:03.866 DEBUG:teuthology.orchestra.run.trial103:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T22:24:03.869 DEBUG:teuthology.orchestra.run.trial190:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T22:24:03.907 INFO:teuthology.orchestra.run.trial103.stdout:[global] 2026-02-22T22:24:03.907 INFO:teuthology.orchestra.run.trial103.stdout: chdir = "" 2026-02-22T22:24:03.908 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:24:03.908 INFO:teuthology.orchestra.run.trial103.stdout: auth supported = cephx 2026-02-22T22:24:03.908 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.908 INFO:teuthology.orchestra.run.trial103.stdout: filestore xattr use omap = true 2026-02-22T22:24:03.908 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.908 INFO:teuthology.orchestra.run.trial103.stdout: mon clock drift allowed = 1.000 2026-02-22T22:24:03.908 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.908 INFO:teuthology.orchestra.run.trial103.stdout: osd crush chooseleaf type = 0 2026-02-22T22:24:03.908 INFO:teuthology.orchestra.run.trial103.stdout: auth debug = true 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: ms die on old message = true 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: ms die on bug = true 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: mon pg warn max object skew = 0 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.909 INFO:teuthology.orchestra.run.trial103.stdout: osd pool default size = 2 2026-02-22T22:24:03.910 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.910 INFO:teuthology.orchestra.run.trial103.stdout: mon osd allow primary affinity = true 2026-02-22T22:24:03.910 INFO:teuthology.orchestra.run.trial103.stdout: mon osd allow pg remap = true 2026-02-22T22:24:03.910 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:24:03.910 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:24:03.910 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on no sortbitwise = false 2026-02-22T22:24:03.910 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:24:03.910 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on too few osds = false 2026-02-22T22:24:03.910 INFO:teuthology.orchestra.run.trial103.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:24:03.910 INFO:teuthology.orchestra.run.trial103.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:24:03.911 INFO:teuthology.orchestra.run.trial103.stdout: mon_allow_pool_size_one = true 2026-02-22T22:24:03.911 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.911 INFO:teuthology.orchestra.run.trial103.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:24:03.911 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.911 INFO:teuthology.orchestra.run.trial103.stdout: osd default data pool replay window = 5 2026-02-22T22:24:03.911 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.911 INFO:teuthology.orchestra.run.trial103.stdout: mon allow pool delete = true 2026-02-22T22:24:03.911 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.911 INFO:teuthology.orchestra.run.trial103.stdout: mon cluster log file level = debug 2026-02-22T22:24:03.911 INFO:teuthology.orchestra.run.trial103.stdout: debug asserts on shutdown = true 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout: mon health detail to clog = false 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout: mon host = 10.20.193.190 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout: mon client directed command retry = 5 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout: mon warn on pool no app = False 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout: ms inject socket failures = 5000 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout: fsid = 56258423-fe54-446e-ae04-77688092fba9 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout:[osd] 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout: osd journal size = 100 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.912 INFO:teuthology.orchestra.run.trial103.stdout: osd scrub load threshold = 5.0 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: osd scrub max interval = 600 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: osd mclock skip benchmark = true 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: osd recover clone overlap = true 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: osd recovery max chunk = 1048576 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: osd debug shutdown = true 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: osd debug op order = true 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: osd debug verify stray on activate = true 2026-02-22T22:24:03.913 INFO:teuthology.orchestra.run.trial103.stdout: osd debug trim objects = true 2026-02-22T22:24:03.914 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.914 INFO:teuthology.orchestra.run.trial103.stdout: osd open classes on start = true 2026-02-22T22:24:03.914 INFO:teuthology.orchestra.run.trial103.stdout: osd debug pg log writeout = true 2026-02-22T22:24:03.914 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.914 INFO:teuthology.orchestra.run.trial103.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:24:03.914 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.914 INFO:teuthology.orchestra.run.trial103.stdout: osd map max advance = 10 2026-02-22T22:24:03.914 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.914 INFO:teuthology.orchestra.run.trial103.stdout: journal zero on create = true 2026-02-22T22:24:03.914 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: filestore apply finisher threads = 3 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: bdev debug aio = true 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: osd debug misdirected ops = true 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: bluefs allocator = hybrid 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: bluestore allocator = hybrid 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: bluestore block size = 96636764160 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: bluestore fsck on mount = True 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: debug bluefs = 1/20 2026-02-22T22:24:03.915 INFO:teuthology.orchestra.run.trial103.stdout: debug bluestore = 1/20 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout: debug ms = 1 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout: debug osd = 20 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout: debug rocksdb = 4/10 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout: mon osd full ratio = 0.9 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout: osd objectstore = bluestore 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout:[mgr] 2026-02-22T22:24:03.916 INFO:teuthology.orchestra.run.trial103.stdout: debug ms = 1 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout: debug mgr = 20 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout: debug mon = 20 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout: debug auth = 20 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout: mgr/telemetry/nag = false 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout:[mon] 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout: debug ms = 1 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout: debug mon = 20 2026-02-22T22:24:03.917 INFO:teuthology.orchestra.run.trial103.stdout: debug paxos = 20 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: debug auth = 20 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: mon data avail warn = 5 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: mon mgr mkfs grace = 240 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: mon osd reporter subtree level = osd 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: mon osd prime pg temp = true 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:24:03.918 INFO:teuthology.orchestra.run.trial103.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout: # 1m isn't quite enough 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:24:03.919 INFO:teuthology.orchestra.run.trial103.stdout:[client] 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout: rgw cache enabled = true 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout: rgw enable ops log = true 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout: rgw enable usage log = true 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout: debug rbd = 20 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout: debug rbd_mirror = 15 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout: log to stderr = False 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout: rbd default features = 125 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout: rbd mirror image policy type = none 2026-02-22T22:24:03.920 INFO:teuthology.orchestra.run.trial103.stdout:[mon.a] 2026-02-22T22:24:03.921 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.0] 2026-02-22T22:24:03.921 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.921 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.921 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.1] 2026-02-22T22:24:03.921 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.921 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.921 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.2] 2026-02-22T22:24:03.921 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.921 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.921 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.3] 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.4] 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.5] 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout:[client.mirror.6] 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.922 INFO:teuthology.orchestra.run.trial103.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.924 INFO:teuthology.orchestra.run.trial190.stdout:[global] 2026-02-22T22:24:03.924 INFO:teuthology.orchestra.run.trial190.stdout: chdir = "" 2026-02-22T22:24:03.924 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:24:03.924 INFO:teuthology.orchestra.run.trial190.stdout: auth supported = cephx 2026-02-22T22:24:03.925 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.925 INFO:teuthology.orchestra.run.trial190.stdout: filestore xattr use omap = true 2026-02-22T22:24:03.925 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.925 INFO:teuthology.orchestra.run.trial190.stdout: mon clock drift allowed = 1.000 2026-02-22T22:24:03.925 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.925 INFO:teuthology.orchestra.run.trial190.stdout: osd crush chooseleaf type = 0 2026-02-22T22:24:03.925 INFO:teuthology.orchestra.run.trial190.stdout: auth debug = true 2026-02-22T22:24:03.925 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.925 INFO:teuthology.orchestra.run.trial190.stdout: ms die on old message = true 2026-02-22T22:24:03.925 INFO:teuthology.orchestra.run.trial190.stdout: ms die on bug = true 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: mon pg warn max object skew = 0 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: osd pool default size = 2 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: mon osd allow primary affinity = true 2026-02-22T22:24:03.926 INFO:teuthology.orchestra.run.trial190.stdout: mon osd allow pg remap = true 2026-02-22T22:24:03.927 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:24:03.927 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:24:03.927 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on no sortbitwise = false 2026-02-22T22:24:03.927 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:24:03.927 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on too few osds = false 2026-02-22T22:24:03.927 INFO:teuthology.orchestra.run.trial190.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:24:03.927 INFO:teuthology.orchestra.run.trial190.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:24:03.927 INFO:teuthology.orchestra.run.trial190.stdout: mon_allow_pool_size_one = true 2026-02-22T22:24:03.927 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.927 INFO:teuthology.orchestra.run.trial190.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: osd default data pool replay window = 5 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: mon allow pool delete = true 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: mon cluster log file level = debug 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: debug asserts on shutdown = true 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: mon health detail to clog = false 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: mon host = 10.20.193.190 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: mon client directed command retry = 5 2026-02-22T22:24:03.928 INFO:teuthology.orchestra.run.trial190.stdout: mon warn on pool no app = False 2026-02-22T22:24:03.929 INFO:teuthology.orchestra.run.trial190.stdout: ms inject socket failures = 5000 2026-02-22T22:24:03.929 INFO:teuthology.orchestra.run.trial190.stdout: fsid = 56258423-fe54-446e-ae04-77688092fba9 2026-02-22T22:24:03.929 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.929 INFO:teuthology.orchestra.run.trial190.stdout:[osd] 2026-02-22T22:24:03.929 INFO:teuthology.orchestra.run.trial190.stdout: osd journal size = 100 2026-02-22T22:24:03.929 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.929 INFO:teuthology.orchestra.run.trial190.stdout: osd scrub load threshold = 5.0 2026-02-22T22:24:03.929 INFO:teuthology.orchestra.run.trial190.stdout: osd scrub max interval = 600 2026-02-22T22:24:03.929 INFO:teuthology.orchestra.run.trial190.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:24:03.929 INFO:teuthology.orchestra.run.trial190.stdout: osd mclock skip benchmark = true 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: osd recover clone overlap = true 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: osd recovery max chunk = 1048576 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: osd debug shutdown = true 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: osd debug op order = true 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: osd debug verify stray on activate = true 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: osd debug trim objects = true 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: osd open classes on start = true 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: osd debug pg log writeout = true 2026-02-22T22:24:03.930 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.931 INFO:teuthology.orchestra.run.trial190.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:24:03.931 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.931 INFO:teuthology.orchestra.run.trial190.stdout: osd map max advance = 10 2026-02-22T22:24:03.931 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.931 INFO:teuthology.orchestra.run.trial190.stdout: journal zero on create = true 2026-02-22T22:24:03.931 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.931 INFO:teuthology.orchestra.run.trial190.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:24:03.931 INFO:teuthology.orchestra.run.trial190.stdout: filestore apply finisher threads = 3 2026-02-22T22:24:03.931 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.931 INFO:teuthology.orchestra.run.trial190.stdout: bdev debug aio = true 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: osd debug misdirected ops = true 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: bluefs allocator = hybrid 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: bluestore allocator = hybrid 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: bluestore block size = 96636764160 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: bluestore fsck on mount = True 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: debug bluefs = 1/20 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: debug bluestore = 1/20 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: debug ms = 1 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: debug osd = 20 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: debug rocksdb = 4/10 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:24:03.932 INFO:teuthology.orchestra.run.trial190.stdout: mon osd full ratio = 0.9 2026-02-22T22:24:03.933 INFO:teuthology.orchestra.run.trial190.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:24:03.933 INFO:teuthology.orchestra.run.trial190.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:24:03.933 INFO:teuthology.orchestra.run.trial190.stdout: osd objectstore = bluestore 2026-02-22T22:24:03.933 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.933 INFO:teuthology.orchestra.run.trial190.stdout:[mgr] 2026-02-22T22:24:03.933 INFO:teuthology.orchestra.run.trial190.stdout: debug ms = 1 2026-02-22T22:24:03.933 INFO:teuthology.orchestra.run.trial190.stdout: debug mgr = 20 2026-02-22T22:24:03.933 INFO:teuthology.orchestra.run.trial190.stdout: debug mon = 20 2026-02-22T22:24:03.933 INFO:teuthology.orchestra.run.trial190.stdout: debug auth = 20 2026-02-22T22:24:03.933 INFO:teuthology.orchestra.run.trial190.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout: mgr/telemetry/nag = false 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout:[mon] 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout: debug ms = 1 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout: debug mon = 20 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout: debug paxos = 20 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout: debug auth = 20 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout: mon data avail warn = 5 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout: mon mgr mkfs grace = 240 2026-02-22T22:24:03.934 INFO:teuthology.orchestra.run.trial190.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: mon osd reporter subtree level = osd 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: mon osd prime pg temp = true 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:24:03.935 INFO:teuthology.orchestra.run.trial190.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout: # 1m isn't quite enough 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout:[client] 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout: rgw cache enabled = true 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout: rgw enable ops log = true 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout: rgw enable usage log = true 2026-02-22T22:24:03.936 INFO:teuthology.orchestra.run.trial190.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:24:03.937 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:24:03.937 INFO:teuthology.orchestra.run.trial190.stdout: debug rbd = 20 2026-02-22T22:24:03.937 INFO:teuthology.orchestra.run.trial190.stdout: debug rbd_mirror = 15 2026-02-22T22:24:03.937 INFO:teuthology.orchestra.run.trial190.stdout: log to stderr = False 2026-02-22T22:24:03.937 INFO:teuthology.orchestra.run.trial190.stdout: rbd default features = 125 2026-02-22T22:24:03.937 INFO:teuthology.orchestra.run.trial190.stdout: rbd mirror image policy type = none 2026-02-22T22:24:03.937 INFO:teuthology.orchestra.run.trial190.stdout:[mon.a] 2026-02-22T22:24:03.937 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.0] 2026-02-22T22:24:03.937 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.937 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.1] 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.2] 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.3] 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.4] 2026-02-22T22:24:03.938 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.939 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.939 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.5] 2026-02-22T22:24:03.939 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.939 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.939 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.6] 2026-02-22T22:24:03.939 INFO:teuthology.orchestra.run.trial190.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:24:03.939 INFO:teuthology.orchestra.run.trial190.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:24:03.940 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-22T22:24:03.941 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-22T22:24:04.004 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T22:24:04.004 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:04.004 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-22T22:24:04.049 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:04.050 DEBUG:teuthology.orchestra.run.trial190:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-22T22:24:04.097 INFO:tasks.ceph:Sending monmap to node ubuntu@trial103.front.sepia.ceph.com 2026-02-22T22:24:04.097 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:24:04.097 DEBUG:teuthology.orchestra.run.trial103:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T22:24:04.097 DEBUG:teuthology.orchestra.run.trial103:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:24:04.111 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:24:04.112 DEBUG:teuthology.orchestra.run.trial103:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:24:04.157 INFO:tasks.ceph:Sending monmap to node ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:24:04.157 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:04.157 DEBUG:teuthology.orchestra.run.trial190:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T22:24:04.158 DEBUG:teuthology.orchestra.run.trial190:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:24:04.170 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:04.170 DEBUG:teuthology.orchestra.run.trial190:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:24:04.217 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T22:24:04.217 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T22:24:04.217 DEBUG:teuthology.orchestra.run.trial103:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T22:24:04.245 INFO:teuthology.orchestra.run.trial103.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T22:24:04.248 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T22:24:04.248 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T22:24:04.249 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-22T22:24:04.315 INFO:teuthology.orchestra.run.trial103.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-22T22:24:04.323 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T22:24:04.342 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T22:24:04.349 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T22:24:04.409 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T22:24:04.417 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T22:24:04.477 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T22:24:04.486 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T22:24:04.545 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T22:24:04.552 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T22:24:04.608 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T22:24:04.617 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T22:24:04.677 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T22:24:04.685 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T22:24:04.741 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T22:24:04.749 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T22:24:04.809 INFO:teuthology.orchestra.run.trial190.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T22:24:04.817 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T22:24:04.817 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial103.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial190.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:24:04.817 DEBUG:teuthology.orchestra.run.trial190:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-22T22:24:04.868 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:24:04.869 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-22T22:24:04.869 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:24:04.869 DEBUG:teuthology.orchestra.run.trial190:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T22:24:04.918 INFO:teuthology.orchestra.run.trial190.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:24:04.918 INFO:teuthology.orchestra.run.trial190.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:24:04.918 INFO:teuthology.orchestra.run.trial190.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:24:04.918 INFO:teuthology.orchestra.run.trial190.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T22:24:04.919 INFO:teuthology.orchestra.run.trial190.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:24:04.919 INFO:teuthology.orchestra.run.trial190.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:24:04.919 INFO:teuthology.orchestra.run.trial190.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:24:04.919 INFO:teuthology.orchestra.run.trial190.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:24:04.919 INFO:teuthology.orchestra.run.trial190.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:24:04.919 INFO:teuthology.orchestra.run.trial190.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:24:05.493 INFO:teuthology.orchestra.run.trial190.stdout:Discarding blocks...Done. 2026-02-22T22:24:05.494 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial190.front.sepia.ceph.com -o noatime 2026-02-22T22:24:05.495 DEBUG:teuthology.orchestra.run.trial190:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-22T22:24:05.550 DEBUG:teuthology.orchestra.run.trial190:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-22T22:24:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:24:05.599 INFO:teuthology.orchestra.run.trial190.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T22:24:05.600 DEBUG:teuthology.orchestra.run.trial190:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-22T22:24:05.649 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:24:05.649 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-22T22:24:05.649 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:24:05.649 DEBUG:teuthology.orchestra.run.trial190:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T22:24:05.702 INFO:teuthology.orchestra.run.trial190.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:24:05.702 INFO:teuthology.orchestra.run.trial190.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:24:05.703 INFO:teuthology.orchestra.run.trial190.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:24:05.703 INFO:teuthology.orchestra.run.trial190.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T22:24:05.703 INFO:teuthology.orchestra.run.trial190.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:24:05.703 INFO:teuthology.orchestra.run.trial190.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:24:05.703 INFO:teuthology.orchestra.run.trial190.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:24:05.703 INFO:teuthology.orchestra.run.trial190.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:24:05.703 INFO:teuthology.orchestra.run.trial190.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:24:05.703 INFO:teuthology.orchestra.run.trial190.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:24:06.267 INFO:teuthology.orchestra.run.trial190.stdout:Discarding blocks...Done. 2026-02-22T22:24:06.268 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial190.front.sepia.ceph.com -o noatime 2026-02-22T22:24:06.269 DEBUG:teuthology.orchestra.run.trial190:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-22T22:24:06.326 DEBUG:teuthology.orchestra.run.trial190:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-22T22:24:06.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:24:06.375 INFO:teuthology.orchestra.run.trial190.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T22:24:06.376 DEBUG:teuthology.orchestra.run.trial190:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-22T22:24:06.425 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:24:06.425 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-22T22:24:06.425 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:24:06.426 DEBUG:teuthology.orchestra.run.trial190:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T22:24:06.478 INFO:teuthology.orchestra.run.trial190.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:24:06.479 INFO:teuthology.orchestra.run.trial190.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:24:06.479 INFO:teuthology.orchestra.run.trial190.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:24:06.479 INFO:teuthology.orchestra.run.trial190.stdout: = reflink=1 bigtime=0 inobtcount=0 2026-02-22T22:24:06.479 INFO:teuthology.orchestra.run.trial190.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:24:06.479 INFO:teuthology.orchestra.run.trial190.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:24:06.479 INFO:teuthology.orchestra.run.trial190.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:24:06.479 INFO:teuthology.orchestra.run.trial190.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:24:06.479 INFO:teuthology.orchestra.run.trial190.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:24:06.480 INFO:teuthology.orchestra.run.trial190.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:24:07.041 INFO:teuthology.orchestra.run.trial190.stdout:Discarding blocks...Done. 2026-02-22T22:24:07.043 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial190.front.sepia.ceph.com -o noatime 2026-02-22T22:24:07.043 DEBUG:teuthology.orchestra.run.trial190:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-22T22:24:07.098 DEBUG:teuthology.orchestra.run.trial190:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-22T22:24:07.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:24:07.147 INFO:teuthology.orchestra.run.trial190.stderr:sudo: /sbin/restorecon: command not found 2026-02-22T22:24:07.148 DEBUG:teuthology.orchestra.run.trial190:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:24:07.211 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:07.206+0000 7f25435338c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-22T22:24:07.212 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:07.206+0000 7f25435338c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-22T22:24:07.212 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:07.206+0000 7f25435338c0 -1 bdev(0x55e0322a3800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-22T22:24:07.212 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:07.206+0000 7f25435338c0 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-22T22:24:10.307 DEBUG:teuthology.orchestra.run.trial190:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-22T22:24:10.358 DEBUG:teuthology.orchestra.run.trial190:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:24:10.422 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:10.418+0000 7fb0382078c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-22T22:24:10.422 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:10.418+0000 7fb0382078c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-22T22:24:10.422 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:10.418+0000 7fb0382078c0 -1 bdev(0x55a6ed77d800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-22T22:24:10.422 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:10.418+0000 7fb0382078c0 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-22T22:24:13.580 DEBUG:teuthology.orchestra.run.trial190:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-22T22:24:13.630 DEBUG:teuthology.orchestra.run.trial190:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:24:13.694 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:13.690+0000 7f187eaee8c0 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-22T22:24:13.695 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:13.690+0000 7f187eaee8c0 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-22T22:24:13.695 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:13.690+0000 7f187eaee8c0 -1 bdev(0x558740ac9800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-22T22:24:13.695 INFO:teuthology.orchestra.run.trial190.stderr:2026-02-22T22:24:13.690+0000 7f187eaee8c0 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-22T22:24:16.801 DEBUG:teuthology.orchestra.run.trial190:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-22T22:24:16.850 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T22:24:16.850 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:24:16.850 DEBUG:teuthology.orchestra.run.trial103:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-22T22:24:16.862 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:16.862 DEBUG:teuthology.orchestra.run.trial190:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-22T22:24:16.902 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:16.902 DEBUG:teuthology.orchestra.run.trial190:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-22T22:24:16.954 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:16.954 DEBUG:teuthology.orchestra.run.trial190:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-22T22:24:17.006 DEBUG:teuthology.orchestra.run.trial103:> set -ex 2026-02-22T22:24:17.006 DEBUG:teuthology.orchestra.run.trial103:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-22T22:24:17.012 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:17.012 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-22T22:24:17.054 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:17.054 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-22T22:24:17.101 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:17.101 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-22T22:24:17.149 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:17.150 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-22T22:24:17.197 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:17.197 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-22T22:24:17.245 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:17.245 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-22T22:24:17.289 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:17.289 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-22T22:24:17.333 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:17.333 DEBUG:teuthology.orchestra.run.trial190:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-22T22:24:17.377 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T22:24:17.377 DEBUG:teuthology.orchestra.run.trial190:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-22T22:24:17.423 INFO:teuthology.orchestra.run.trial190.stdout:[mgr.x] 2026-02-22T22:24:17.424 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpSVOYDhAAs7LDBNjGzGqmTk/z0/frhA== 2026-02-22T22:24:17.424 INFO:teuthology.orchestra.run.trial190.stdout:[osd.0] 2026-02-22T22:24:17.424 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAHgptpuJV9DBAAodDI/6fzLav2EpjcLJVLoQ== 2026-02-22T22:24:17.424 INFO:teuthology.orchestra.run.trial190.stdout:[osd.1] 2026-02-22T22:24:17.424 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAKgptpoYoSGRAA4FVt1CTRrZBpRrWDnNfA/A== 2026-02-22T22:24:17.424 INFO:teuthology.orchestra.run.trial190.stdout:[osd.2] 2026-02-22T22:24:17.424 INFO:teuthology.orchestra.run.trial190.stdout: key = AQANgptpTURBKRAAkQnNK6UdeQTe7GeNNMr+kw== 2026-02-22T22:24:17.424 INFO:teuthology.orchestra.run.trial190.stdout:[client.0] 2026-02-22T22:24:17.424 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpIs+8EhAAdk4sQ4Nc3sK90MNb/wOYCg== 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror] 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptps0dSFBAAERvdmIRUwAclrFsqpV/XJA== 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.0] 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpIBRUGBAA60Ghas1qKjlU/+1j5bCMwA== 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.1] 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpU6BiHBAAUPYTzyqdetIMQOQcQejbmg== 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.2] 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpOhhrIBAAPci4JsbzXThmfPyouvD6NQ== 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.3] 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpl7ozJBAAMpl3klokRStWu98MNIHAaA== 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.4] 2026-02-22T22:24:17.425 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpEaNCKBAAKNKwjzf7q9+MIv4FINMuFA== 2026-02-22T22:24:17.426 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.5] 2026-02-22T22:24:17.426 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptp9KcULBAAvS05C2FOFfHZ0hVQfdKNGw== 2026-02-22T22:24:17.426 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.6] 2026-02-22T22:24:17.426 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpe10jMBAAbGfF+6Xpl0MjMuUbThUlCg== 2026-02-22T22:24:17.427 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T22:24:17.491 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:24:17.553 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:24:17.618 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:24:17.681 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:24:17.746 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:24:17.811 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:24:17.875 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:24:17.938 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:24:18.002 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:24:18.066 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:24:18.130 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:24:18.194 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:24:18.258 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T22:24:18.258 DEBUG:teuthology.orchestra.run.trial190:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-22T22:24:18.309 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-22T22:24:18.381 DEBUG:teuthology.orchestra.run.trial190:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-22T22:24:18.433 DEBUG:teuthology.orchestra.run.trial190:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:24:18.484 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-22T22:24:18.485 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-22T22:24:18.485 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-22T22:24:18.526 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-22T22:24:18.526 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-22T22:24:18.526 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-22T22:24:18.526 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-22T22:24:18.527 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-22T22:24:18.527 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T22:24:18.527 DEBUG:teuthology.orchestra.run.trial190:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-22T22:24:18.656 INFO:teuthology.orchestra.run.trial190.stderr:adjusted tunables profile to default 2026-02-22T22:24:18.666 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T22:24:18.666 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-22T22:24:18.666 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:18.667 DEBUG:teuthology.orchestra.run.trial190:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-22T22:24:18.677 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:18.677 DEBUG:teuthology.orchestra.run.trial190:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-22T22:24:18.679 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:18.676+0000 7f9ed068c280 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T22:24:18.704 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:18.700+0000 7f9ed068c280 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T22:24:18.730 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:24:18.730 DEBUG:teuthology.orchestra.run.trial190:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-22T22:24:18.739 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:18.736+0000 7f9ed068c280 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T22:24:18.782 DEBUG:teuthology.orchestra.run.trial190:> sudo ceph --cluster cluster2 osd new de5fe4e0-5530-4614-b01f-ba3053c3eff6 0 2026-02-22T22:24:18.803 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:18.800+0000 7f9ed068c280 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T22:24:18.868 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:18.864+0000 7f9ed068c280 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:24:18.904 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:18.900+0000 7f9ed068c280 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T22:24:18.923 INFO:teuthology.orchestra.run.trial190.stdout:0 2026-02-22T22:24:18.933 DEBUG:teuthology.orchestra.run.trial190:> sudo ceph --cluster cluster2 osd new 05eb91fe-01a8-497c-aa22-da7f38c6ce3f 1 2026-02-22T22:24:18.971 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:18.968+0000 7f9ed068c280 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T22:24:19.031 INFO:teuthology.orchestra.run.trial190.stdout:1 2026-02-22T22:24:19.041 DEBUG:teuthology.orchestra.run.trial190:> sudo ceph --cluster cluster2 osd new 392cacdc-46dc-4995-a7ff-7eef33867ec2 2 2026-02-22T22:24:19.132 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:19.128+0000 7f9ed068c280 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T22:24:19.138 INFO:teuthology.orchestra.run.trial190.stdout:2 2026-02-22T22:24:19.149 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-22T22:24:19.149 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-22T22:24:19.150 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-22T22:24:19.150 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-22T22:24:19.150 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-22T22:24:19.151 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-22T22:24:19.151 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-22T22:24:19.151 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-22T22:24:19.152 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-22T22:24:19.152 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:24:19.245 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:19.245 INFO:teuthology.orchestra.run.trial190.stdout:{"epoch":5,"fsid":"56258423-fe54-446e-ae04-77688092fba9","created":"2026-02-22T22:24:18.613141+0000","modified":"2026-02-22T22:24:19.135781+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:24:19.135781+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"de5fe4e0-5530-4614-b01f-ba3053c3eff6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"05eb91fe-01a8-497c-aa22-da7f38c6ce3f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"392cacdc-46dc-4995-a7ff-7eef33867ec2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:24:19.258 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-22T22:24:19.258 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T22:24:19.370 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:19.368+0000 7f9ed068c280 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T22:24:19.457 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:19.452+0000 7f9ed068c280 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T22:24:19.482 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:19.480+0000 7f9ed068c280 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T22:24:19.510 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:19.508+0000 7f9ed068c280 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T22:24:19.540 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:19.536+0000 7f9ed068c280 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T22:24:19.560 DEBUG:teuthology.orchestra.run.trial190:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:24:19.634 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:19.632+0000 7f9ed068c280 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:24:19.657 INFO:teuthology.misc.health.trial190.stdout: 2026-02-22T22:24:19.657 INFO:teuthology.misc.health.trial190.stdout:{"epoch":5,"fsid":"56258423-fe54-446e-ae04-77688092fba9","created":"2026-02-22T22:24:18.613141+0000","modified":"2026-02-22T22:24:19.135781+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:24:19.135781+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"de5fe4e0-5530-4614-b01f-ba3053c3eff6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"05eb91fe-01a8-497c-aa22-da7f38c6ce3f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"392cacdc-46dc-4995-a7ff-7eef33867ec2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:24:19.666 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:24:19.801 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:ERROR:root:Module 'xmltodict' is not installed. 2026-02-22T22:24:19.948 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:19.944+0000 7f9ed068c280 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T22:24:19.979 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:19.976+0000 7f9ed068c280 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T22:24:20.051 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:20.048+0000 7f9ed068c280 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T22:24:20.075 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:20.072+0000 7f9ed068c280 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T22:24:20.100 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:20.096+0000 7f9ed068c280 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T22:24:20.125 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:20.124+0000 7f9ed068c280 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T22:24:20.202 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:/usr/lib/python3/dist-packages/scipy/__init__.py:67: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T22:24:20.202 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T22:24:20.203 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr: from numpy import show_config as show_numpy_config 2026-02-22T22:24:20.208 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:20.204+0000 7f9ed068c280 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T22:24:20.241 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:20.240+0000 7f9ed068c280 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T22:24:20.278 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:20.276+0000 7f9ed068c280 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T22:24:20.670 INFO:tasks.ceph.cluster2.osd.0.trial190.stderr:2026-02-22T22:24:20.666+0000 7f62e58038c0 -1 Falling back to public interface 2026-02-22T22:24:20.738 INFO:tasks.ceph.cluster2.osd.2.trial190.stderr:2026-02-22T22:24:20.734+0000 7f6f0fcd88c0 -1 Falling back to public interface 2026-02-22T22:24:20.746 INFO:tasks.ceph.cluster2.osd.1.trial190.stderr:2026-02-22T22:24:20.742+0000 7f4c9a8c48c0 -1 Falling back to public interface 2026-02-22T22:24:21.288 INFO:tasks.ceph.cluster2.mgr.x.trial103.stderr:2026-02-22T22:24:21.284+0000 7f9deeed9640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T22:24:24.493 INFO:tasks.ceph.cluster2.osd.0.trial190.stderr:2026-02-22T22:24:24.486+0000 7f62e58038c0 -1 osd.0 0 log_to_monitors true 2026-02-22T22:24:24.918 INFO:tasks.ceph.cluster2.osd.1.trial190.stderr:2026-02-22T22:24:24.914+0000 7f4c9a8c48c0 -1 osd.1 0 log_to_monitors true 2026-02-22T22:24:25.123 INFO:tasks.ceph.cluster2.osd.2.trial190.stderr:2026-02-22T22:24:25.118+0000 7f6f0fcd88c0 -1 osd.2 0 log_to_monitors true 2026-02-22T22:24:25.969 DEBUG:teuthology.orchestra.run.trial190:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:24:26.116 INFO:teuthology.misc.health.trial190.stdout: 2026-02-22T22:24:26.116 INFO:teuthology.misc.health.trial190.stdout:{"epoch":6,"fsid":"56258423-fe54-446e-ae04-77688092fba9","created":"2026-02-22T22:24:18.613141+0000","modified":"2026-02-22T22:24:25.284697+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:24:19.135781+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"de5fe4e0-5530-4614-b01f-ba3053c3eff6","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"05eb91fe-01a8-497c-aa22-da7f38c6ce3f","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"392cacdc-46dc-4995-a7ff-7eef33867ec2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:24:26.125 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:24:26.294 INFO:tasks.ceph.cluster2.osd.0.trial190.stderr:2026-02-22T22:24:26.290+0000 7f62e17ac640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T22:24:26.295 INFO:tasks.ceph.cluster2.osd.1.trial190.stderr:2026-02-22T22:24:26.290+0000 7f4c9686d640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T22:24:26.295 INFO:tasks.ceph.cluster2.osd.2.trial190.stderr:2026-02-22T22:24:26.290+0000 7f6f0bc81640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T22:24:26.310 INFO:tasks.ceph.cluster2.osd.0.trial190.stderr:2026-02-22T22:24:26.306+0000 7f62d95df640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:24:26.322 INFO:tasks.ceph.cluster2.osd.2.trial190.stderr:2026-02-22T22:24:26.318+0000 7f6f03ab4640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:24:26.322 INFO:tasks.ceph.cluster2.osd.1.trial190.stderr:2026-02-22T22:24:26.318+0000 7f4c8e6a0640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:24:32.428 DEBUG:teuthology.orchestra.run.trial190:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:24:32.570 INFO:teuthology.misc.health.trial190.stdout: 2026-02-22T22:24:32.571 INFO:teuthology.misc.health.trial190.stdout:{"epoch":11,"fsid":"56258423-fe54-446e-ae04-77688092fba9","created":"2026-02-22T22:24:18.613141+0000","modified":"2026-02-22T22:24:31.299276+0000","last_up_change":"2026-02-22T22:24:27.289747+0000","last_in_change":"2026-02-22T22:24:19.135781+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:24:28.313150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"de5fe4e0-5530-4614-b01f-ba3053c3eff6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6800","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6801","nonce":1600625483}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6802","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6803","nonce":1600625483}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6806","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6807","nonce":1600625483}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6804","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6805","nonce":1600625483}]},"public_addr":"10.20.193.190:6801/1600625483","cluster_addr":"10.20.193.190:6803/1600625483","heartbeat_back_addr":"10.20.193.190:6807/1600625483","heartbeat_front_addr":"10.20.193.190:6805/1600625483","state":["exists","up"]},{"osd":1,"uuid":"05eb91fe-01a8-497c-aa22-da7f38c6ce3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6816","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6817","nonce":587589839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6818","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6819","nonce":587589839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6822","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6823","nonce":587589839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6820","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6821","nonce":587589839}]},"public_addr":"10.20.193.190:6817/587589839","cluster_addr":"10.20.193.190:6819/587589839","heartbeat_back_addr":"10.20.193.190:6823/587589839","heartbeat_front_addr":"10.20.193.190:6821/587589839","state":["exists","up"]},{"osd":2,"uuid":"392cacdc-46dc-4995-a7ff-7eef33867ec2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6808","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6809","nonce":3480696925}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6810","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6811","nonce":3480696925}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6814","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6815","nonce":3480696925}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6812","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6813","nonce":3480696925}]},"public_addr":"10.20.193.190:6809/3480696925","cluster_addr":"10.20.193.190:6811/3480696925","heartbeat_back_addr":"10.20.193.190:6815/3480696925","heartbeat_front_addr":"10.20.193.190:6813/3480696925","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:24:32.580 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T22:24:32.581 INFO:tasks.ceph:Creating RBD pool 2026-02-22T22:24:32.581 DEBUG:teuthology.orchestra.run.trial190:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-22T22:24:33.310 INFO:teuthology.orchestra.run.trial190.stderr:pool 'rbd' created 2026-02-22T22:24:33.324 DEBUG:teuthology.orchestra.run.trial190:> rbd --cluster cluster2 pool init rbd 2026-02-22T22:24:36.325 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-22T22:24:36.325 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-22T22:24:36.325 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-22T22:24:36.326 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-22T22:24:36.326 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T22:24:36.488 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:36.498 INFO:teuthology.orchestra.run.trial190.stdout:{"epoch":5,"flags":0,"active_gid":4100,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6826","nonce":157225376},{"type":"v1","addr":"10.20.193.103:6827","nonce":157225376}]},"active_addr":"10.20.193.103:6827/157225376","active_change":"2026-02-22T22:24:20.277971+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.103:0","nonce":3901459805}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.103:0","nonce":2306682205}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.103:0","nonce":3295729066}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.103:0","nonce":3859602314}]}]} 2026-02-22T22:24:36.504 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-22T22:24:36.504 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-22T22:24:36.505 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:24:36.647 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:36.648 INFO:teuthology.orchestra.run.trial190.stdout:{"epoch":15,"fsid":"56258423-fe54-446e-ae04-77688092fba9","created":"2026-02-22T22:24:18.613141+0000","modified":"2026-02-22T22:24:36.312664+0000","last_up_change":"2026-02-22T22:24:27.289747+0000","last_in_change":"2026-02-22T22:24:19.135781+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:24:28.313150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:24:32.728976+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"de5fe4e0-5530-4614-b01f-ba3053c3eff6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6800","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6801","nonce":1600625483}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6802","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6803","nonce":1600625483}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6806","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6807","nonce":1600625483}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6804","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6805","nonce":1600625483}]},"public_addr":"10.20.193.190:6801/1600625483","cluster_addr":"10.20.193.190:6803/1600625483","heartbeat_back_addr":"10.20.193.190:6807/1600625483","heartbeat_front_addr":"10.20.193.190:6805/1600625483","state":["exists","up"]},{"osd":1,"uuid":"05eb91fe-01a8-497c-aa22-da7f38c6ce3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6816","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6817","nonce":587589839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6818","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6819","nonce":587589839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6822","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6823","nonce":587589839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6820","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6821","nonce":587589839}]},"public_addr":"10.20.193.190:6817/587589839","cluster_addr":"10.20.193.190:6819/587589839","heartbeat_back_addr":"10.20.193.190:6823/587589839","heartbeat_front_addr":"10.20.193.190:6821/587589839","state":["exists","up"]},{"osd":2,"uuid":"392cacdc-46dc-4995-a7ff-7eef33867ec2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6808","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6809","nonce":3480696925}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6810","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6811","nonce":3480696925}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6814","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6815","nonce":3480696925}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6812","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6813","nonce":3480696925}]},"public_addr":"10.20.193.190:6809/3480696925","cluster_addr":"10.20.193.190:6811/3480696925","heartbeat_back_addr":"10.20.193.190:6815/3480696925","heartbeat_front_addr":"10.20.193.190:6813/3480696925","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:24:36.658 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-22T22:24:36.658 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:24:36.806 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:36.806 INFO:teuthology.orchestra.run.trial190.stdout:{"epoch":15,"fsid":"56258423-fe54-446e-ae04-77688092fba9","created":"2026-02-22T22:24:18.613141+0000","modified":"2026-02-22T22:24:36.312664+0000","last_up_change":"2026-02-22T22:24:27.289747+0000","last_in_change":"2026-02-22T22:24:19.135781+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:24:28.313150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:24:32.728976+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"de5fe4e0-5530-4614-b01f-ba3053c3eff6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6800","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6801","nonce":1600625483}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6802","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6803","nonce":1600625483}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6806","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6807","nonce":1600625483}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6804","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6805","nonce":1600625483}]},"public_addr":"10.20.193.190:6801/1600625483","cluster_addr":"10.20.193.190:6803/1600625483","heartbeat_back_addr":"10.20.193.190:6807/1600625483","heartbeat_front_addr":"10.20.193.190:6805/1600625483","state":["exists","up"]},{"osd":1,"uuid":"05eb91fe-01a8-497c-aa22-da7f38c6ce3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6816","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6817","nonce":587589839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6818","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6819","nonce":587589839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6822","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6823","nonce":587589839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6820","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6821","nonce":587589839}]},"public_addr":"10.20.193.190:6817/587589839","cluster_addr":"10.20.193.190:6819/587589839","heartbeat_back_addr":"10.20.193.190:6823/587589839","heartbeat_front_addr":"10.20.193.190:6821/587589839","state":["exists","up"]},{"osd":2,"uuid":"392cacdc-46dc-4995-a7ff-7eef33867ec2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6808","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6809","nonce":3480696925}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6810","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6811","nonce":3480696925}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6814","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6815","nonce":3480696925}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6812","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6813","nonce":3480696925}]},"public_addr":"10.20.193.190:6809/3480696925","cluster_addr":"10.20.193.190:6811/3480696925","heartbeat_back_addr":"10.20.193.190:6815/3480696925","heartbeat_front_addr":"10.20.193.190:6813/3480696925","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:24:36.816 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-22T22:24:36.817 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-22T22:24:36.818 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-22T22:24:36.900 INFO:teuthology.orchestra.run.trial190.stdout:34359738371 2026-02-22T22:24:36.900 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T22:24:36.901 INFO:teuthology.orchestra.run.trial190.stdout:34359738372 2026-02-22T22:24:36.902 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T22:24:36.907 INFO:teuthology.orchestra.run.trial190.stdout:34359738372 2026-02-22T22:24:36.908 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T22:24:37.053 INFO:teuthology.orchestra.run.trial190.stdout:34359738370 2026-02-22T22:24:37.065 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.0 2026-02-22T22:24:37.098 INFO:teuthology.orchestra.run.trial190.stdout:34359738371 2026-02-22T22:24:37.100 INFO:teuthology.orchestra.run.trial190.stdout:34359738371 2026-02-22T22:24:37.110 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.2 2026-02-22T22:24:37.112 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.1 2026-02-22T22:24:38.066 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T22:24:38.111 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T22:24:38.112 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T22:24:38.215 INFO:teuthology.orchestra.run.trial190.stdout:34359738370 2026-02-22T22:24:38.226 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.0 2026-02-22T22:24:38.256 INFO:teuthology.orchestra.run.trial190.stdout:34359738371 2026-02-22T22:24:38.268 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.1 2026-02-22T22:24:38.269 INFO:teuthology.orchestra.run.trial190.stdout:34359738371 2026-02-22T22:24:38.280 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.2 2026-02-22T22:24:39.227 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T22:24:39.269 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T22:24:39.281 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T22:24:39.374 INFO:teuthology.orchestra.run.trial190.stdout:34359738372 2026-02-22T22:24:39.386 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738372 for osd.0 2026-02-22T22:24:39.386 DEBUG:teuthology.parallel:result is None 2026-02-22T22:24:39.418 INFO:teuthology.orchestra.run.trial190.stdout:34359738372 2026-02-22T22:24:39.431 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.1 2026-02-22T22:24:39.431 DEBUG:teuthology.parallel:result is None 2026-02-22T22:24:39.471 INFO:teuthology.orchestra.run.trial190.stdout:34359738373 2026-02-22T22:24:39.483 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738373 for osd.2 2026-02-22T22:24:39.483 DEBUG:teuthology.parallel:result is None 2026-02-22T22:24:39.484 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T22:24:39.484 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:24:39.671 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:39.671 INFO:teuthology.orchestra.run.trial190.stderr:dumped all 2026-02-22T22:24:39.680 INFO:teuthology.orchestra.run.trial190.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T22:24:39.568745+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82440,"kb_used_data":1544,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033080,"statfs":{"total":289910292480,"available":289825873920,"internally_reserved":0,"allocated":1581056,"data_stored":1291889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.257176"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.318108+0000","last_change":"2026-02-22T22:24:36.318108+0000","last_active":"2026-02-22T22:24:36.318108+0000","last_peered":"2026-02-22T22:24:36.318108+0000","last_clean":"2026-02-22T22:24:36.318108+0000","last_became_active":"2026-02-22T22:24:34.316382+0000","last_became_peered":"2026-02-22T22:24:34.316382+0000","last_unstale":"2026-02-22T22:24:36.318108+0000","last_undegraded":"2026-02-22T22:24:36.318108+0000","last_fullsized":"2026-02-22T22:24:36.318108+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:30:56.453533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057076399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.318553+0000","last_change":"2026-02-22T22:24:36.318553+0000","last_active":"2026-02-22T22:24:36.318553+0000","last_peered":"2026-02-22T22:24:36.318553+0000","last_clean":"2026-02-22T22:24:36.318553+0000","last_became_active":"2026-02-22T22:24:34.315465+0000","last_became_peered":"2026-02-22T22:24:34.315465+0000","last_unstale":"2026-02-22T22:24:36.318553+0000","last_undegraded":"2026-02-22T22:24:36.318553+0000","last_fullsized":"2026-02-22T22:24:36.318553+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:27:00.271117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00102524,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.318688+0000","last_change":"2026-02-22T22:24:36.318688+0000","last_active":"2026-02-22T22:24:36.318688+0000","last_peered":"2026-02-22T22:24:36.318688+0000","last_clean":"2026-02-22T22:24:36.318688+0000","last_became_active":"2026-02-22T22:24:34.316505+0000","last_became_peered":"2026-02-22T22:24:34.316505+0000","last_unstale":"2026-02-22T22:24:36.318688+0000","last_undegraded":"2026-02-22T22:24:36.318688+0000","last_fullsized":"2026-02-22T22:24:36.318688+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:54.581297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001096383,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.318328+0000","last_change":"2026-02-22T22:24:36.318328+0000","last_active":"2026-02-22T22:24:36.318328+0000","last_peered":"2026-02-22T22:24:36.318328+0000","last_clean":"2026-02-22T22:24:36.318328+0000","last_became_active":"2026-02-22T22:24:34.316336+0000","last_became_peered":"2026-02-22T22:24:34.316336+0000","last_unstale":"2026-02-22T22:24:36.318328+0000","last_undegraded":"2026-02-22T22:24:36.318328+0000","last_fullsized":"2026-02-22T22:24:36.318328+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:42:20.387771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079384900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.316873+0000","last_change":"2026-02-22T22:24:36.315539+0000","last_active":"2026-02-22T22:24:36.316873+0000","last_peered":"2026-02-22T22:24:36.316873+0000","last_clean":"2026-02-22T22:24:36.316873+0000","last_became_active":"2026-02-22T22:24:34.315494+0000","last_became_peered":"2026-02-22T22:24:34.315494+0000","last_unstale":"2026-02-22T22:24:36.316873+0000","last_undegraded":"2026-02-22T22:24:36.316873+0000","last_fullsized":"2026-02-22T22:24:36.316873+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:57:09.391401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278449,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:37.193025+0000","last_change":"2026-02-22T22:24:37.193154+0000","last_active":"2026-02-22T22:24:37.193025+0000","last_peered":"2026-02-22T22:24:37.193025+0000","last_clean":"2026-02-22T22:24:37.193025+0000","last_became_active":"2026-02-22T22:24:34.315699+0000","last_became_peered":"2026-02-22T22:24:34.315699+0000","last_unstale":"2026-02-22T22:24:37.193025+0000","last_undegraded":"2026-02-22T22:24:37.193025+0000","last_fullsized":"2026-02-22T22:24:37.193025+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:06:14.131443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028952900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:37.192990+0000","last_change":"2026-02-22T22:24:37.193148+0000","last_active":"2026-02-22T22:24:37.192990+0000","last_peered":"2026-02-22T22:24:37.192990+0000","last_clean":"2026-02-22T22:24:37.192990+0000","last_became_active":"2026-02-22T22:24:34.315765+0000","last_became_peered":"2026-02-22T22:24:34.315765+0000","last_unstale":"2026-02-22T22:24:37.192990+0000","last_undegraded":"2026-02-22T22:24:37.192990+0000","last_fullsized":"2026-02-22T22:24:37.192990+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:23:11.125424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278349,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.318064+0000","last_change":"2026-02-22T22:24:36.318064+0000","last_active":"2026-02-22T22:24:36.318064+0000","last_peered":"2026-02-22T22:24:36.318064+0000","last_clean":"2026-02-22T22:24:36.318064+0000","last_became_active":"2026-02-22T22:24:34.315206+0000","last_became_peered":"2026-02-22T22:24:34.315206+0000","last_unstale":"2026-02-22T22:24:36.318064+0000","last_undegraded":"2026-02-22T22:24:36.318064+0000","last_fullsized":"2026-02-22T22:24:36.318064+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:48:25.916897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056075500000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.317223+0000","last_change":"2026-02-22T22:24:30.304133+0000","last_active":"2026-02-22T22:24:36.317223+0000","last_peered":"2026-02-22T22:24:36.317223+0000","last_clean":"2026-02-22T22:24:36.317223+0000","last_became_active":"2026-02-22T22:24:30.303257+0000","last_became_peered":"2026-02-22T22:24:30.303257+0000","last_unstale":"2026-02-22T22:24:36.317223+0000","last_undegraded":"2026-02-22T22:24:36.317223+0000","last_fullsized":"2026-02-22T22:24:36.317223+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_clean_scrub_stamp":"2026-02-22T22:24:29.295633+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:57:58.047355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27088,"kb_used_data":120,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344752,"statfs":{"total":96636764160,"available":96609026048,"internally_reserved":0,"allocated":122880,"data_stored":34665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27672,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344168,"statfs":{"total":96636764160,"available":96608428032,"internally_reserved":0,"allocated":729088,"data_stored":628612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27680,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344160,"statfs":{"total":96636764160,"available":96608419840,"internally_reserved":0,"allocated":729088,"data_stored":628612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:24:39.681 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:24:39.833 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:39.833 INFO:teuthology.orchestra.run.trial190.stderr:dumped all 2026-02-22T22:24:39.845 INFO:teuthology.orchestra.run.trial190.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T22:24:39.568745+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82440,"kb_used_data":1544,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033080,"statfs":{"total":289910292480,"available":289825873920,"internally_reserved":0,"allocated":1581056,"data_stored":1291889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.257176"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.318108+0000","last_change":"2026-02-22T22:24:36.318108+0000","last_active":"2026-02-22T22:24:36.318108+0000","last_peered":"2026-02-22T22:24:36.318108+0000","last_clean":"2026-02-22T22:24:36.318108+0000","last_became_active":"2026-02-22T22:24:34.316382+0000","last_became_peered":"2026-02-22T22:24:34.316382+0000","last_unstale":"2026-02-22T22:24:36.318108+0000","last_undegraded":"2026-02-22T22:24:36.318108+0000","last_fullsized":"2026-02-22T22:24:36.318108+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:30:56.453533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057076399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.318553+0000","last_change":"2026-02-22T22:24:36.318553+0000","last_active":"2026-02-22T22:24:36.318553+0000","last_peered":"2026-02-22T22:24:36.318553+0000","last_clean":"2026-02-22T22:24:36.318553+0000","last_became_active":"2026-02-22T22:24:34.315465+0000","last_became_peered":"2026-02-22T22:24:34.315465+0000","last_unstale":"2026-02-22T22:24:36.318553+0000","last_undegraded":"2026-02-22T22:24:36.318553+0000","last_fullsized":"2026-02-22T22:24:36.318553+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:27:00.271117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00102524,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.318688+0000","last_change":"2026-02-22T22:24:36.318688+0000","last_active":"2026-02-22T22:24:36.318688+0000","last_peered":"2026-02-22T22:24:36.318688+0000","last_clean":"2026-02-22T22:24:36.318688+0000","last_became_active":"2026-02-22T22:24:34.316505+0000","last_became_peered":"2026-02-22T22:24:34.316505+0000","last_unstale":"2026-02-22T22:24:36.318688+0000","last_undegraded":"2026-02-22T22:24:36.318688+0000","last_fullsized":"2026-02-22T22:24:36.318688+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:54.581297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001096383,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.318328+0000","last_change":"2026-02-22T22:24:36.318328+0000","last_active":"2026-02-22T22:24:36.318328+0000","last_peered":"2026-02-22T22:24:36.318328+0000","last_clean":"2026-02-22T22:24:36.318328+0000","last_became_active":"2026-02-22T22:24:34.316336+0000","last_became_peered":"2026-02-22T22:24:34.316336+0000","last_unstale":"2026-02-22T22:24:36.318328+0000","last_undegraded":"2026-02-22T22:24:36.318328+0000","last_fullsized":"2026-02-22T22:24:36.318328+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:42:20.387771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079384900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.316873+0000","last_change":"2026-02-22T22:24:36.315539+0000","last_active":"2026-02-22T22:24:36.316873+0000","last_peered":"2026-02-22T22:24:36.316873+0000","last_clean":"2026-02-22T22:24:36.316873+0000","last_became_active":"2026-02-22T22:24:34.315494+0000","last_became_peered":"2026-02-22T22:24:34.315494+0000","last_unstale":"2026-02-22T22:24:36.316873+0000","last_undegraded":"2026-02-22T22:24:36.316873+0000","last_fullsized":"2026-02-22T22:24:36.316873+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:57:09.391401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278449,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:37.193025+0000","last_change":"2026-02-22T22:24:37.193154+0000","last_active":"2026-02-22T22:24:37.193025+0000","last_peered":"2026-02-22T22:24:37.193025+0000","last_clean":"2026-02-22T22:24:37.193025+0000","last_became_active":"2026-02-22T22:24:34.315699+0000","last_became_peered":"2026-02-22T22:24:34.315699+0000","last_unstale":"2026-02-22T22:24:37.193025+0000","last_undegraded":"2026-02-22T22:24:37.193025+0000","last_fullsized":"2026-02-22T22:24:37.193025+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:06:14.131443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028952900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:37.192990+0000","last_change":"2026-02-22T22:24:37.193148+0000","last_active":"2026-02-22T22:24:37.192990+0000","last_peered":"2026-02-22T22:24:37.192990+0000","last_clean":"2026-02-22T22:24:37.192990+0000","last_became_active":"2026-02-22T22:24:34.315765+0000","last_became_peered":"2026-02-22T22:24:34.315765+0000","last_unstale":"2026-02-22T22:24:37.192990+0000","last_undegraded":"2026-02-22T22:24:37.192990+0000","last_fullsized":"2026-02-22T22:24:37.192990+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:23:11.125424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278349,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.318064+0000","last_change":"2026-02-22T22:24:36.318064+0000","last_active":"2026-02-22T22:24:36.318064+0000","last_peered":"2026-02-22T22:24:36.318064+0000","last_clean":"2026-02-22T22:24:36.318064+0000","last_became_active":"2026-02-22T22:24:34.315206+0000","last_became_peered":"2026-02-22T22:24:34.315206+0000","last_unstale":"2026-02-22T22:24:36.318064+0000","last_undegraded":"2026-02-22T22:24:36.318064+0000","last_fullsized":"2026-02-22T22:24:36.318064+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:48:25.916897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056075500000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":109,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:24:36.317223+0000","last_change":"2026-02-22T22:24:30.304133+0000","last_active":"2026-02-22T22:24:36.317223+0000","last_peered":"2026-02-22T22:24:36.317223+0000","last_clean":"2026-02-22T22:24:36.317223+0000","last_became_active":"2026-02-22T22:24:30.303257+0000","last_became_peered":"2026-02-22T22:24:30.303257+0000","last_unstale":"2026-02-22T22:24:36.317223+0000","last_undegraded":"2026-02-22T22:24:36.317223+0000","last_fullsized":"2026-02-22T22:24:36.317223+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_clean_scrub_stamp":"2026-02-22T22:24:29.295633+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:57:58.047355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27088,"kb_used_data":120,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344752,"statfs":{"total":96636764160,"available":96609026048,"internally_reserved":0,"allocated":122880,"data_stored":34665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27672,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344168,"statfs":{"total":96636764160,"available":96608428032,"internally_reserved":0,"allocated":729088,"data_stored":628612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27680,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344160,"statfs":{"total":96636764160,"available":96608419840,"internally_reserved":0,"allocated":729088,"data_stored":628612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:24:39.846 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T22:24:39.847 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-22T22:24:39.847 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-22T22:24:39.847 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-22T22:24:40.018 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:24:40.018 INFO:teuthology.orchestra.run.trial190.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T22:24:40.030 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-22T22:24:40.031 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T22:24:40.042 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T22:24:40.043 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:24:40.043 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:40.220 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.4] 2026-02-22T22:24:40.221 INFO:teuthology.orchestra.run.trial190.stdout: key = AQDfgZtpvf/sABAAv1A9ajbXYrwrHUX+DUbbZg== 2026-02-22T22:24:40.221 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:40.221 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:40.221 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.4 2026-02-22T22:24:40.231 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:40.418 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.5] 2026-02-22T22:24:40.418 INFO:teuthology.orchestra.run.trial190.stdout: key = AQDfgZtpW5e+BBAASc0hf5UVy8G8Ydq7JhlhGQ== 2026-02-22T22:24:40.418 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:40.418 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:40.418 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.5 2026-02-22T22:24:40.429 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:40.606 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.6] 2026-02-22T22:24:40.606 INFO:teuthology.orchestra.run.trial190.stdout: key = AQDfgZtpeKCQCBAAxFB8yaynqUBFBWkwLyFCXg== 2026-02-22T22:24:40.606 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:40.606 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:40.606 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.6 2026-02-22T22:24:40.617 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:40.799 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.4] 2026-02-22T22:24:40.799 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpEaNCKBAAKNKwjzf7q9+MIv4FINMuFA== 2026-02-22T22:24:40.799 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:40.799 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:40.799 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.4 2026-02-22T22:24:40.810 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:40.990 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.5] 2026-02-22T22:24:40.990 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptp9KcULBAAvS05C2FOFfHZ0hVQfdKNGw== 2026-02-22T22:24:40.990 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:40.990 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:40.991 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.5 2026-02-22T22:24:41.001 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:41.182 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.6] 2026-02-22T22:24:41.182 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpe10jMBAAbGfF+6Xpl0MjMuUbThUlCg== 2026-02-22T22:24:41.182 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:41.182 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:41.183 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.6 2026-02-22T22:24:41.194 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T22:24:41.217 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T22:24:41.217 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:24:41.218 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:41.396 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror] 2026-02-22T22:24:41.396 INFO:teuthology.orchestra.run.trial190.stdout: key = AQDegZtpt3IXKBAAeLd6y0dqxL5/hlELgIlIDA== 2026-02-22T22:24:41.396 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T22:24:41.397 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:41.397 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror 2026-02-22T22:24:41.407 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:41.586 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.0] 2026-02-22T22:24:41.586 INFO:teuthology.orchestra.run.trial190.stdout: key = AQDegZtpRS9+LBAAgtVC4L92btWj0SAiket+EA== 2026-02-22T22:24:41.587 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:41.587 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:41.587 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.0 2026-02-22T22:24:41.597 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:41.774 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.1] 2026-02-22T22:24:41.774 INFO:teuthology.orchestra.run.trial190.stdout: key = AQDegZtpjlCjMBAA1wq5iUdSmBBjzJLWW8ukfg== 2026-02-22T22:24:41.775 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:41.775 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:41.775 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.1 2026-02-22T22:24:41.785 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:41.965 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.2] 2026-02-22T22:24:41.965 INFO:teuthology.orchestra.run.trial190.stdout: key = AQDegZtpcNPDNBAAeDfsVrgH68tWW6zqt80+6Q== 2026-02-22T22:24:41.965 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:41.965 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:41.965 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.2 2026-02-22T22:24:41.977 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:42.153 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.3] 2026-02-22T22:24:42.153 INFO:teuthology.orchestra.run.trial190.stdout: key = AQDegZtpfrW0OBAAfwpS5cE/72S5R/mRzXUddA== 2026-02-22T22:24:42.153 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:42.153 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:42.153 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.3 2026-02-22T22:24:42.163 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:42.346 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror] 2026-02-22T22:24:42.346 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptps0dSFBAAERvdmIRUwAclrFsqpV/XJA== 2026-02-22T22:24:42.346 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T22:24:42.346 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:42.347 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror 2026-02-22T22:24:42.358 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:42.538 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.0] 2026-02-22T22:24:42.538 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpIBRUGBAA60Ghas1qKjlU/+1j5bCMwA== 2026-02-22T22:24:42.538 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:42.538 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:42.538 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.0 2026-02-22T22:24:42.549 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:42.729 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.1] 2026-02-22T22:24:42.729 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpU6BiHBAAUPYTzyqdetIMQOQcQejbmg== 2026-02-22T22:24:42.729 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:42.729 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:42.730 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.1 2026-02-22T22:24:42.740 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:42.917 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.2] 2026-02-22T22:24:42.918 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpOhhrIBAAPci4JsbzXThmfPyouvD6NQ== 2026-02-22T22:24:42.918 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:42.918 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:42.918 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.2 2026-02-22T22:24:42.928 DEBUG:teuthology.orchestra.run.trial190:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:24:43.110 INFO:teuthology.orchestra.run.trial190.stdout:[client.mirror.3] 2026-02-22T22:24:43.111 INFO:teuthology.orchestra.run.trial190.stdout: key = AQAEgptpl7ozJBAAMpl3klokRStWu98MNIHAaA== 2026-02-22T22:24:43.111 INFO:teuthology.orchestra.run.trial190.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:24:43.111 INFO:teuthology.orchestra.run.trial190.stdout: caps osd = "profile rbd" 2026-02-22T22:24:43.111 INFO:teuthology.orchestra.run.trial190.stderr:updated caps for client.mirror.3 2026-02-22T22:24:43.122 INFO:teuthology.run_tasks:Running task workunit... 2026-02-22T22:24:43.149 INFO:tasks.workunit:Pulling workunits from ref f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:24:43.149 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-22T22:24:43.149 DEBUG:teuthology.orchestra.run.trial190:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:24:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:24:43.155 INFO:teuthology.orchestra.run.trial190.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-22T22:24:43.156 DEBUG:teuthology.orchestra.run.trial190:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:24:43.201 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:24:43.202 DEBUG:teuthology.orchestra.run.trial190:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-22T22:24:43.249 INFO:tasks.workunit:timeout=6h 2026-02-22T22:24:43.249 INFO:tasks.workunit:cleanup=True 2026-02-22T22:24:43.250 DEBUG:teuthology.orchestra.run.trial190:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:24:43.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-22T22:25:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Note: switching to 'f90f4eaebb28b619cf76e80cbbe997656da69579'. 2026-02-22T22:25:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: 2026-02-22T22:25:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-22T22:25:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-22T22:25:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:state without impacting any branches by switching back to a branch. 2026-02-22T22:25:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: 2026-02-22T22:25:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: git switch -c 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Or undo this operation with: 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: git switch - 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: 2026-02-22T22:25:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:HEAD is now at f90f4eaebb2 Merge branch 'wip-74712-squid' of https://github.com/idryomov/ceph into wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:25:05.581 DEBUG:teuthology.orchestra.run.trial190:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-22T22:25:05.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-22T22:25:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T22:25:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-22T22:25:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-22T22:25:05.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-22T22:25:05.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T22:25:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T22:25:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-22T22:25:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T22:25:05.808 DEBUG:teuthology.orchestra.run.trial190:> set -ex 2026-02-22T22:25:05.809 DEBUG:teuthology.orchestra.run.trial190:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-22T22:25:05.855 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_ha.sh on cluster1.client.mirror... 2026-02-22T22:25:05.860 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_ha.sh... 2026-02-22T22:25:05.860 DEBUG:teuthology.orchestra.run.trial190:workunit test rbd/rbd_mirror_ha.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f90f4eaebb28b619cf76e80cbbe997656da69579 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_ha.sh 2026-02-22T22:25:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ RBD_MIRROR_INSTANCES=7 2026-02-22T22:25:05.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_ha.sh 2026-02-22T22:25:05.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ type xmlstarlet 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ XMLSTARLET=xmlstarlet 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ RBD_MIRROR_INSTANCES=7 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CLUSTER1=cluster1 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CLUSTER2=cluster2 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ POOL=mirror 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ PARENT_POOL=mirror_parent 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ NS1=ns1 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ NS2=ns2 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ TEMPDIR= 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ID=mirror 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,journaling 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ RBD_MIRROR_MODE=journal 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ MIRROR_POOL_MODE=pool 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' journal = snapshot ']' 2026-02-22T22:25:05.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-22T22:25:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS='--id mirror' 2026-02-22T22:25:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ LAST_MIRROR_INSTANCE=6 2026-02-22T22:25:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_ha.sh 2026-02-22T22:25:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-22T22:25:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-22T22:25:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_BIN=. 2026-02-22T22:25:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_SRC=. 2026-02-22T22:25:05.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-22T22:25:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 -gt 0 ']' 2026-02-22T22:25:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ setup 2026-02-22T22:25:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local c 2026-02-22T22:25:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-22T22:25:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ setup_tempdir 2026-02-22T22:25:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mktemp -d 2026-02-22T22:25:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ TEMPDIR=/tmp/tmp.ek6CoRRxsq 2026-02-22T22:25:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z 1 ']' 2026-02-22T22:25:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ setup_pools cluster1 cluster2 2026-02-22T22:25:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:25:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local remote_cluster=cluster2 2026-02-22T22:25:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local mon_map_file 2026-02-22T22:25:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local mon_addr 2026-02-22T22:25:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local admin_key_file 2026-02-22T22:25:05.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local uuid 2026-02-22T22:25:05.917 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:25:05.917 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-22T22:25:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:pool 'mirror' already exists 2026-02-22T22:25:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:25:06.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-22T22:25:07.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:pool 'mirror_parent' already exists 2026-02-22T22:25:07.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:25:07.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-22T22:25:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:25:10.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-22T22:25:13.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:13.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:25:13.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror pool 2026-02-22T22:25:13.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-22T22:25:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-22T22:25:13.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-22T22:25:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-22T22:25:13.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 pool 2026-02-22T22:25:13.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-22T22:25:13.434 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 pool 2026-02-22T22:25:13.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z ']' 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z ']' 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ peer_add cluster1 mirror cluster2 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local client_cluster=cluster2 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local remote_cluster=cluster2 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local uuid_var_name 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local error_code 2026-02-22T22:25:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local peer_uuid 2026-02-22T22:25:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T22:25:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set +e 2026-02-22T22:25:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2026-02-22T22:25:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ peer_uuid=9c592ea6-c5db-4622-bc1c-f3aa6c613a7d 2026-02-22T22:25:13.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ error_code=0 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set -e 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 0 -eq 0 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ peer_add cluster1 mirror_parent cluster2 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror_parent 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local client_cluster=cluster2 2026-02-22T22:25:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local remote_cluster=cluster2 2026-02-22T22:25:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local uuid_var_name 2026-02-22T22:25:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local error_code 2026-02-22T22:25:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local peer_uuid 2026-02-22T22:25:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T22:25:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set +e 2026-02-22T22:25:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2026-02-22T22:25:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ peer_uuid=f1d5c308-7067-41cb-af07-b4e23b47c54b 2026-02-22T22:25:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ error_code=0 2026-02-22T22:25:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set -e 2026-02-22T22:25:13.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 0 -eq 0 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ setup_pools cluster2 cluster1 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local remote_cluster=cluster1 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local mon_map_file 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local mon_addr 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local admin_key_file 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local uuid 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:25:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-22T22:25:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:pool 'mirror' already exists 2026-02-22T22:25:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:25:14.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-22T22:25:15.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:pool 'mirror_parent' already exists 2026-02-22T22:25:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:25:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-22T22:25:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:25:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-22T22:25:21.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:25:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror pool 2026-02-22T22:25:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-22T22:25:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-22T22:25:21.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-22T22:25:21.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-22T22:25:21.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 pool 2026-02-22T22:25:21.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-22T22:25:21.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 pool 2026-02-22T22:25:21.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z ']' 2026-02-22T22:25:21.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z ']' 2026-02-22T22:25:21.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ peer_add cluster2 mirror cluster1 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local client_cluster=cluster1 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local remote_cluster=cluster1 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local uuid_var_name 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local error_code 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local peer_uuid 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set +e 2026-02-22T22:25:21.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2026-02-22T22:25:21.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ peer_uuid=1b28b876-6c48-4a7e-a35f-67380ddadb6f 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ error_code=0 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set -e 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 0 -eq 0 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ peer_add cluster2 mirror_parent cluster1 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror_parent 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local client_cluster=cluster1 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local remote_cluster=cluster1 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local uuid_var_name 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local error_code 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local peer_uuid 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set +e 2026-02-22T22:25:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ peer_uuid=7f2569d4-74a4-438f-9516-70cb7a2e1594 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ error_code=0 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set -e 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 0 -eq 0 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: start first daemon instance and test replay' 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:25:21.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:25:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:25:21 TEST: start first daemon instance and test replay 2026-02-22T22:25:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:25:21 TEST: start first daemon instance and test replay 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:0 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:25:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image1=test1 2026-02-22T22:25:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test1 2026-02-22T22:25:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:25:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test1 2026-02-22T22:25:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=0 2026-02-22T22:25:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 2026-02-22T22:25:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:25:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:25:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test1 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:25:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:25:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:25:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:25:22.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:25:22.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:25:22.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:25:22.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:25:22.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:25:22.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:22.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:25:22.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:25:22.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:22.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:25:22.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:25:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:25:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:25:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:25:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:25:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:25:22.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:25:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:25:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:25:23.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:25:23.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:23.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:25:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:25:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:25:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:25:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:25:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:25:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:25:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:25:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:25:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:25:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:25:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:25:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:25:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:25:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:25:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:25:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:25:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:25:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:25:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:25:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:24.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 2 2026-02-22T22:25:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:25:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:25:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:25:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:25:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:25:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:25:26.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:25:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:25:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:25:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:25:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:25:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:25:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:25:26.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:26.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:26.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:26.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:26.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:25:26.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:25:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:25:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:25:26.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:25:26.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:25:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:25:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:25:26.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:25:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:25:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:25:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:25:26.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:25:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:25:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:25:26.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 8333.48 bytes/sec: 33 MiB/s 2026-02-22T22:25:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:25:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:25:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:25:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:25:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:25:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:25:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:25:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:25:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:25:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:25:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:25:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:25:26.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:26.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:25:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:25:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:25:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:26.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:26.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:25:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:25:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:25:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:25:26.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:25:26.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:25:26.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:25:26.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:26.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:25:26.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:25:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:25:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:25:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:25:26.591 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:25:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:26.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:26.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:25:26.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:26.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:26.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:25:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:25:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:25:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:25:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:25:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:25:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:25:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:25:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:25:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:25:26.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:25:26.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:25:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:25:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=1, entry_tid=1' '!=' '' 2026-02-22T22:25:26.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:25:26.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=1, entry_tid=1' = '' 2026-02-22T22:25:26.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:25:26.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:26.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:25:26.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:25:26.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:25:26.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:25:26.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:25:26.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:25:26.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=1 2026-02-22T22:25:26.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:26.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:25:26.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:25:26.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:25:26.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:25:26.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:25:26.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:25:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=1 2026-02-22T22:25:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 1 -a 101 -le 1 2026-02-22T22:25:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:25:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:25:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:25:27.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:25:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:25:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:25:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:25:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:25:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:25:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:25:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:25:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:25:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:25:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:25:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:25:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:25:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:25:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:25:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:25:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:25:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:25:27.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:25:27.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:27.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:27.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:25:27.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:27.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:27.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:27.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:25:27.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:27.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:27.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:27.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:25:27.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:25:27.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:25:27.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:25:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:25:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:25:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:25:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:25:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:25:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:25:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:27.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:27.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:25:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:25:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:25:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:25:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:25:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:25:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:25:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:25:28.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:25:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:25:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:25:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:25:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:25:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:25:28.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:25:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:25:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:25:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:25:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:25:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:25:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:25:25' 2026-02-22T22:25:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:25:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:25:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:25:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:25:28.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:25:28.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:25:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:25:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:25:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:25:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:25:25' 2026-02-22T22:25:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:25:25' 2026-02-22T22:25:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:25:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:25:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:25:25' 2026-02-22T22:25:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:25:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:25:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:25:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING142664269mirror.0trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42664269mirror.0trial1902026-02-22 22:25:25' 2026-02-22T22:25:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:25:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:25:28.288 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:25:25' 2026-02-22T22:25:28.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:25:28.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:25:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:25:28.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:25:28.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:25:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:25:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:25:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:25:28.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:25:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:25:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:28.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:25:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:25:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:25:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:25:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:25:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:25:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:25:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:25:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:25:25' 2026-02-22T22:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:25:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:25:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:25:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:25:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:25:29.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:25:29.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:25:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:25:29.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:25:29.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:25:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:25:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:25:29.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:25:29.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK1test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:25:25' 2026-02-22T22:25:29.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:25:29.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:25:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:25:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:25:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:25:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:25:29.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:25:29.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:25:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:25:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:25:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:25:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:25:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:25:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:25:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:25:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:25:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:25:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:25:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:25:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:25:29.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:25:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:25:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:25:29.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:25:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:25:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: release leader and wait it is reacquired' 2026-02-22T22:25:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:25:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:25:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:25:29 TEST: release leader and wait it is reacquired 2026-02-22T22:25:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:25:29 TEST: release leader and wait it is reacquired 2026-02-22T22:25:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 mirror 2026-02-22T22:25:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n mirror 2026-02-22T22:25:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:29.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:29.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:29.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:29.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 mirror_parent 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror_parent 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n mirror_parent 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror_parent cluster2 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent cluster2 2026-02-22T22:25:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:25:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ release_leader mirror 2026-02-22T22:25:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror leader release' 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n mirror 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror leader release mirror cluster2' 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror leader release mirror cluster2 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror leader release mirror cluster2 2026-02-22T22:25:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader_released 2026-02-22T22:25:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local i 2026-02-22T22:25:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 0 2026-02-22T22:25:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 10 2026-02-22T22:25:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in `seq 10` 2026-02-22T22:25:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:29.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:29.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:29.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:29.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 mirror_parent 2026-02-22T22:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror_parent 2026-02-22T22:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n mirror_parent 2026-02-22T22:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror_parent cluster2 2026-02-22T22:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:30.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent cluster2 2026-02-22T22:25:30.118 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:25:30.118 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:25:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:25:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:25:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:25:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:25:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:31.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:31.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:31.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:31.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:25:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:25:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:31.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:25:31.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:25:31.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:25:31.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:31.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:31.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:31.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:25:31.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:25:31.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:25:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:25:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:25:31.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:25:31.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:31.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:31.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:31.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:25:31.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:25:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:25:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:25:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:25:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:25:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:25:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:25:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:25:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:25:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:25:31.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:31.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:25:31.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:25:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:25:31.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:25:31.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:31.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:25:31.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:31.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:31.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:31.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:25:31.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:25:31.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:31.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:31.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:31.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:25:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:25:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:25:31.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:31.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:31.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:25:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:25:31.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:25:31.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:31.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:31.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:31.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:25:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:25:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:25:32.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:25:32.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:32.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:32.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:32.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:32.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:25:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:25:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:25:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:25:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:25:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:25:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:25:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:32.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:32.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:32.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:25:32.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:25:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:25:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:25:32.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:32.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:32.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:32.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:25:32.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:25:32.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:32.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:32.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:25:32.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:25:32.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:25:32.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:32.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:32.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:32.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:25:32.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:32.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:25:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:25:32.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:25:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:25:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:25:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:25:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:25:32.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:25:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:25:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:25:32.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:32.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:32.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:32.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:25:32.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:25:32.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:32.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:32.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:25:32.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:25:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:25:33.032 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:25:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 2 2026-02-22T22:25:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:25:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:35.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:35.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:35.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:25:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:25:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:25:35.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:35.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:25:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:25:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:25:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:25:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:25:35.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:35.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:35.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:35.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:25:35.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:35.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:25:35.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:25:35.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:25:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:25:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:25:35.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:35.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:35.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:35.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:25:35.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:25:35.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:25:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:35.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:25:35.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:35.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:35.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:35.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:25:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:25:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:25:35.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:25:35.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:35.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:35.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:35.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:25:35.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:25:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:25:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:25:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:25:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:25:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:25:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:25:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:25:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:25:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:25:35.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:35.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:35.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:35.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:25:35.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:35.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:35.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:35.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:25:35.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:35.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:35.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:35.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:25:35.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:35.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:35.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:25:35.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:35.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:25:35.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:25:35.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:25:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:25:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:35.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:35.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:35.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:25:35.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:35.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:35.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:25:35.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 4 2026-02-22T22:25:39.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:25:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:39.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:39.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:39.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:39.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:39.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:25:39.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:25:39.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:25:39.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:25:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:39.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:25:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:25:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:25:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:25:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:25:39.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:25:39.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:39.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:25:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:25:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:25:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:25:39.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:39.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:25:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:25:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:25:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:25:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:25:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:25:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:25:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:25:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:25:39.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:25:39.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:39.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:39.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:39.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:25:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:25:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:25:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:25:39.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:39.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:39.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:39.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:25:39.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:39.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:25:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:25:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:25:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:25:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:25:39.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:39.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:25:39.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:25:39.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 4 2026-02-22T22:25:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:25:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:43.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:43.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:43.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:43.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:43.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:25:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=0 2026-02-22T22:25:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ release_leader 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror leader release' 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror leader release 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:44.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:44.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:44.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:44.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror leader release 2026-02-22T22:25:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader_released 2026-02-22T22:25:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local i 2026-02-22T22:25:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 0 2026-02-22T22:25:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 10 2026-02-22T22:25:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in `seq 10` 2026-02-22T22:25:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:44.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:44.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ expect_failure '' is_leader 0 mirror_parent 2026-02-22T22:25:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local expected= 2026-02-22T22:25:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local out=/tmp/tmp.ek6CoRRxsq/expect_failure.out 2026-02-22T22:25:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 mirror_parent 2026-02-22T22:25:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:25:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:25:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:25:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:25:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:25:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:25:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:45.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:45.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:45.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:45.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:45.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:25:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:25:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:25:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:25:45.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:25:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:25:45.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:25:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:45.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:25:45.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:45.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:25:45.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:45.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:45.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:45.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:45.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:45.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:45.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:25:45.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:25:45.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:25:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:25:45.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:45.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:25:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:25:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:25:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:25:45.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:45.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:45.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:45.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:25:45.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:25:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:25:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:45.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:25:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:25:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:25:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:25:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:45.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:25:45.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:25:45.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:45.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:45.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:45.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:25:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:25:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:25:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:25:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:45.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:25:45.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:45.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:45.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:25:45.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:25:45.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:45.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:25:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:25:45.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:25:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:25:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:25:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:25:45.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:45.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:45.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:25:45.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:25:45.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:25:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:25:45.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:25:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:25:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:25:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:25:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:25:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:25:45.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:25:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:25:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:25:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:46.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:25:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:25:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:25:46.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:46.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:46.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:46.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:25:46.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:46.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:25:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:25:46.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:25:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:25:46.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:46.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:46.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:25:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:46.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:25:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:25:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:46.902 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:25:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:25:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:25:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:25:46.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:25:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:25:46.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:25:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:25:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:25:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:25:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:25:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:25:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:25:46.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:46.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:46.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:46.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:25:46.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:46.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:25:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:25:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:25:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:25:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:25:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:25:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:25:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:25:46.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:46.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:46.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:46.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:25:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:25:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:25:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:25:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:25:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:47.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:47.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:47.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:25:47.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:25:47.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:25:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:47.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:25:47.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:47.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:25:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:25:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:25:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:25:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:47.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:25:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:25:47.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:25:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:47.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:25:47.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:25:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 2 2026-02-22T22:25:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:25:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:25:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:25:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:25:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:49.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:25:49.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:49.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:25:49.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:25:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:25:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:25:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:25:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:25:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:25:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:25:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:49.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:25:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:25:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:25:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:25:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:25:49.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:25:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:25:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:25:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:25:49.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:25:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:25:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:25:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:25:49.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:25:49.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:49.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:49.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:25:49.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:25:49.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:25:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:25:49.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:49.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:49.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:49.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:25:49.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:25:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:25:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:25:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:25:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:25:49.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:49.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:49.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:25:49.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:49.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:49.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:25:49.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 4 2026-02-22T22:25:53.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:25:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:53.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:53.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:53.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:53.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:53.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:53.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:53.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:25:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:25:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:25:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:25:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:25:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:25:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:53.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:25:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:25:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:25:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:25:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:25:53.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:53.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:25:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:25:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:25:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:25:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:25:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:25:53.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:25:53.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:25:53.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:25:53.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:25:53.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:25:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:25:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:53.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:53.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:25:53.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:25:53.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:53.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:53.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:25:53.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:25:53.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:53.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:25:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:25:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:25:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:25:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:25:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:25:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:25:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:25:53.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:53.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:53.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:25:53.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:25:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:53.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:53.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:53.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:25:53.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:25:53.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:53.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:53.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:25:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:25:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:25:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:25:53.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:53.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:25:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:25:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:25:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:25:53.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:25:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:25:53.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:25:54.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:25:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:25:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:25:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:25:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:25:54.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:54.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:54.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:25:54.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:25:54.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:25:54.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:54.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:54.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:25:54.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:25:54.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:25:54.100 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:25:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:25:54.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 4 2026-02-22T22:25:58.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:25:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:25:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:25:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:25:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:58.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:58.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:58.118 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:25:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:25:58.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=0 2026-02-22T22:25:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: start second daemon instance and test replay' 2026-02-22T22:25:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:25:58.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:25:58 TEST: start second daemon instance and test replay 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:25:58 TEST: start second daemon instance and test replay 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:1 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:25:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:25:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:25:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:25:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:25:58.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image2=test2 2026-02-22T22:25:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test2 2026-02-22T22:25:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:25:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:25:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:25:58.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test2 2026-02-22T22:25:58.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test2 2026-02-22T22:25:58.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:25:58.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:25:58.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test1 2026-02-22T22:25:58.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:25:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:25:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:25:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:25:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:25:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:25:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:25:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:25:59.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:25:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:25:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:25:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:25:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:25:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:25:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:59.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:25:59.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:25:59.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:59.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:25:59.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:25:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:25:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:25:59.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:25:59.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:25:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:25:59.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:25:59.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:25:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:25:59.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 8333.48 bytes/sec: 33 MiB/s 2026-02-22T22:25:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:25:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:25:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:25:59.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:25:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:25:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:25:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:25:59.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:25:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:25:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:25:59.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:25:59.591 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:25:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:25:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:25:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:25:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:25:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:25:59.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:25:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:25:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:25:59.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:25:59.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:25:59.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:59.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:59.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:59.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:59.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:59.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:59.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:59.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:25:59.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:25:59.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:25:59.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:59.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:59.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:59.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:25:59.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:25:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:25:59.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:25:59.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:25:59.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:59.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:25:59.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:25:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:25:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:25:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:25:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:25:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:25:59.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:25:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:25:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:25:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:25:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:25:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:25:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:25:59.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:25:59.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:25:59.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:59.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:25:59.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:25:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:25:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:25:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:25:59.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:25:59.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:25:59.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' '' 2026-02-22T22:25:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:25:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' = '' 2026-02-22T22:25:59.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:25:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:25:59.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:25:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=3 2026-02-22T22:25:59.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:25:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:25:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=2 2026-02-22T22:25:59.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:25:59.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:25:59.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:25:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:25:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:25:59.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:25:59.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:25:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:25:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 3 = 2 -a 101 -le 101 2026-02-22T22:25:59.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:25:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:25:59.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:25:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:25:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:25:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:25:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:25:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:25:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:25:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:25:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:25:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:59.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:25:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:25:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:25:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:25:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:25:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:25:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:25:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:25:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:25:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:25:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:25:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:25:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:25:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:25:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:26:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:00.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:00.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:00.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:00.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:00.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:00.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:00.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:00.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:00.113 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:26:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:26:00.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:26:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:26:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:26:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:26:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:26:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:00.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:26:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:26:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:00.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:00.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:26:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:00.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:00.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:00.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:00.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:00.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:00.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:00.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:00.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:00.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:00.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:00.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:00.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:26:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:26:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:01.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:01.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:01.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:01.467 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:01.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:01.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:01.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:01.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:01.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:01.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:01.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:01.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:01.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:01.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:01.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:01.585 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:01.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:01.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:01.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:01.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:01.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:01.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:01.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:01.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:01.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:01.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:01.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:01.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:26:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:26:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:01.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:26:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:01.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:02.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:26:02.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:26:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:26:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:26:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:26:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}} 2026-02-22T22:26:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:26:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:25:55' 2026-02-22T22:26:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:26:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:26:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:26:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}} 2026-02-22T22:26:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:26:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:26:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:26:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:25:55' 2026-02-22T22:26:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:25:55' 2026-02-22T22:26:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:25:55' 2026-02-22T22:26:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGWARNINGWARNING242664269mirror.0trial19019.2.3-689-gf90f4eaetrueOK4284mirror.1trial19019.2.3-689-gf90f4eaefalseWARNINGnot reporting statustest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}42664269mirror.0trial1902026-02-22 22:25:55test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42664269mirror.0trial1902026-02-22 22:26:01' 2026-02-22T22:26:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:02.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:26:02.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:25:55' 2026-02-22T22:26:02.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:26:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:26:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:26:02.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:26:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:26:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:26:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:26:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:02.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:26:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:03.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:03.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:26:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:26:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:26:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:26:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:26:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:26:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:26:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:26:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}} 2026-02-22T22:26:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:25:55' 2026-02-22T22:26:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:26:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:26:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:26:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:26:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:26:03.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:26:03.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:03.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:03.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:26:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:26:03.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:03.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK2test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:25:55test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:26:01' 2026-02-22T22:26:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:26:03.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:03.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:26:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:26:03.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:03.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:26:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:26:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:26:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:26:03.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:26:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:26:03.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:03.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:26:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:26:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:26:03.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:03.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:03.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:03.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:26:03.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:03.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:04.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:26:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test2 2026-02-22T22:26:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test2 started 2026-02-22T22:26:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:26:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:26:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:05.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test2 started 2026-02-22T22:26:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:26:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:26:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:26:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test2 2026-02-22T22:26:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:26:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:26:05.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:05.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:05.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:26:05.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:26:05.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:05.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:26:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:26:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test2 2026-02-22T22:26:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:26:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:05.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:26:05.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:26:05.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test2 2026-02-22T22:26:05.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:26:05.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:26:05.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:26:05.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:26:05.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:26:05.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:26:05.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:26:05.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:05.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T22:26:05.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:05.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:05.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:05.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:26:05.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:26:05.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:05.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:26:05.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:26:05.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:26:05.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:26:05.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 8333.48 bytes/sec: 33 MiB/s 2026-02-22T22:26:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test2 2026-02-22T22:26:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:26:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:05.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:26:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test2 2026-02-22T22:26:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:26:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:26:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:26:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:05.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test2 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:26:05.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:05.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:05.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:26:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:26:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:05.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:05.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:05.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test2 2026-02-22T22:26:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:05.607 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:05.607 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:05.607 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:05.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:05.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:05.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:05.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:05.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:05.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:26:05.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:05.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:26:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:05.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:26:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:26:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:26:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=1 2026-02-22T22:26:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:26:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=3 2026-02-22T22:26:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:26:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test2 2026-02-22T22:26:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:05.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:05.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:05.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:05.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test2 2026-02-22T22:26:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:05.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:05.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:06.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:06.001 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:06.001 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:06.001 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=9], [object_number=0, tag_tid=2, entry_tid=8], [object_number=3, tag_tid=2, entry_tid=7], [object_number=2, tag_tid=2, entry_tid=6]]], state=connected] 2026-02-22T22:26:06.003 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:06.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:06.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:06.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=9], [object_number=0, tag_tid=2, entry_tid=8], [object_number=3, tag_tid=2, entry_tid=7], [object_number=2, tag_tid=2, entry_tid=6]]], state=connected] 2026-02-22T22:26:06.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=9' 2026-02-22T22:26:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=9' 2026-02-22T22:26:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=9' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=9' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=9' 2026-02-22T22:26:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:26:06.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=9' 2026-02-22T22:26:06.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:06.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=2 2026-02-22T22:26:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:06.046 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:26:06.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=9' 2026-02-22T22:26:06.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:06.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=9 2026-02-22T22:26:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 2 -a 101 -le 9 2026-02-22T22:26:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test2 2026-02-22T22:26:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:06.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:26:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:26:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:06.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:06.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:06.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:06.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test2 2026-02-22T22:26:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:06.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:06.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:06.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:06.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:06.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:06.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:26:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test2 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:26:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:26:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:06.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:07.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test2 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:26:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:26:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:07.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test2 2026-02-22T22:26:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test2: 2026-02-22T22:26:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0 2026-02-22T22:26:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:26:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:01' 2026-02-22T22:26:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:26:07.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:26:07.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:26:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:26:07.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:26:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:26:07.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:07.461 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:01' 2026-02-22T22:26:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test2 up+replaying '2026-02-22 22:26:01' 2026-02-22T22:26:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:26:01' 2026-02-22T22:26:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:07.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING242664269mirror.0trial19019.2.3-689-gf90f4eaetrueOK42844287mirror.1trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}42664269mirror.0trial1902026-02-22 22:25:55test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42664269mirror.0trial1902026-02-22 22:26:01' 2026-02-22T22:26:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/last_update' 2026-02-22T22:26:07.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:01' 2026-02-22T22:26:07.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/state' 2026-02-22T22:26:07.512 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:26:07.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:26:07.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:26:07.514 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:26:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:26:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test2 down+unknown 2026-02-22T22:26:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:07.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:07.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:08.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test2 down+unknown '' '' 2026-02-22T22:26:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test2 2026-02-22T22:26:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test2: 2026-02-22T22:26:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0 2026-02-22T22:26:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:26:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:26:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:26:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:26:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:26:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:01' 2026-02-22T22:26:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:26:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:26:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:26:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:26:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:26:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:26:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:08.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test2 down+unknown '' 2026-02-22T22:26:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:26:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK2test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:25:55test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:26:01' 2026-02-22T22:26:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/last_update' 2026-02-22T22:26:08.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:08.601 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/state' 2026-02-22T22:26:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:26:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:08.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:26:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:26:08.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:08.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:08.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test2 2026-02-22T22:26:08.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:08.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:26:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test2.export 2026-02-22T22:26:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test2.export 2026-02-22T22:26:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export 2026-02-22T22:26:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test2.export 2026-02-22T22:26:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test2.export 2026-02-22T22:26:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test2 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export 2026-02-22T22:26:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:08.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test2 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:08.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: release leader and test it is acquired by secondary' 2026-02-22T22:26:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:26:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:26:08.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:26:08 TEST: release leader and test it is acquired by secondary 2026-02-22T22:26:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:26:08 TEST: release leader and test it is acquired by secondary 2026-02-22T22:26:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 mirror 2026-02-22T22:26:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:26:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n mirror 2026-02-22T22:26:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:26:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:08.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:26:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:08.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:08.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:08.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:08.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:26:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:26:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 mirror_parent 2026-02-22T22:26:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:26:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror_parent 2026-02-22T22:26:09.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n mirror_parent 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror_parent cluster2 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:09.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:09.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:09.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent cluster2 2026-02-22T22:26:09.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:26:09.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ release_leader mirror 2026-02-22T22:26:09.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:09.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror leader release' 2026-02-22T22:26:09.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n mirror 2026-02-22T22:26:09.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror leader release mirror cluster2' 2026-02-22T22:26:09.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror leader release mirror cluster2 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:09.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror leader release mirror cluster2 2026-02-22T22:26:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader_released 2026-02-22T22:26:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local i 2026-02-22T22:26:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 0 2026-02-22T22:26:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 10 2026-02-22T22:26:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in `seq 10` 2026-02-22T22:26:09.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:26:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:26:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:26:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:26:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:26:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:26:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:26:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:26:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:26:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:10.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:26:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:26:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:26:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:26:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:26:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:26:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:10.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:10.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:10.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:10.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:10.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:10.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:26:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:26:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=1 2026-02-22T22:26:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test2 2026-02-22T22:26:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:26:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:26:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:1 mirror test1 2026-02-22T22:26:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:10.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:1 mirror test1 started 2026-02-22T22:26:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:26:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:26:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:10.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:1 mirror test1 started 2026-02-22T22:26:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:26:11.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:26:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:26:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:26:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:1 2026-02-22T22:26:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:26:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=1 2026-02-22T22:26:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:26:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:26:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:11.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:11.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:26:11.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:26:11.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:26:11.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:26:11.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:11.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T22:26:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:26:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:26:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:26:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:26:12.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:26:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:26:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:12.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:12.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:26:12.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:26:12.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:12.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:12.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:12.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:12.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T22:26:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:26:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:26:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:26:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 2 2026-02-22T22:26:14.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:14.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:26:14.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:26:14.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:14.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:26:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:26:14.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:26:14.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:26:14.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:14.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:14.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:14.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:14.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T22:26:14.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:26:14.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:26:14.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:26:14.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:26:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:26:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:26:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:26:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:26:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:26:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:26:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:26:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:26:14.654 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:26:14.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:26:14.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:1 cluster2 mirror test1 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:1 cluster2 mirror test1 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:26:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:26:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:26:14.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:14.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:14.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:14.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:26:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:14.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:14.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:14.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:14.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:14.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:14.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' '!=' '' 2026-02-22T22:26:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' = '' 2026-02-22T22:26:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:15.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=4 2026-02-22T22:26:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:15.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=3 2026-02-22T22:26:15.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:15.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:15.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:15.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:26:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:26:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 4 = 3 -a 101 -le 101 2026-02-22T22:26:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:26:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:26:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:26:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:15.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:15.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:15.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:15.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:26:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:15.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:15.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:15.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:15.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:15.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:15.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:15.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' '!=' 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:26:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:26:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:26:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:26:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:15.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:15.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:26:15.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:15.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:15.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:15.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:15.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:15.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:15.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:15.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:15.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' '!=' 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:15.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:26:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:26:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:26:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:26:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:16.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:16.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:16.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:16.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:16.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:26:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:16.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:16.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:16.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:16.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:16.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:16.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:16.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:16.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:16.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:16.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:26:16.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:16.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:16.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:17.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:17.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:17.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:17.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:26:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:26:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:26:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}} 2026-02-22T22:26:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.1 on trial190 2026-02-22T22:26:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:13' 2026-02-22T22:26:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:26:17.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:26:17.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:26:17.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}} 2026-02-22T22:26:17.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.1 on trial190' ']' 2026-02-22T22:26:17.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.1 on trial190' 2026-02-22T22:26:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.1 on trial190 2026-02-22T22:26:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:13' 2026-02-22T22:26:17.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:26:13' 2026-02-22T22:26:17.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:17.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:17.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:17.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:17.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:26:13' 2026-02-22T22:26:17.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:17.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:17.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING242664269mirror.0trial19019.2.3-689-gf90f4eaefalseOK42844287mirror.1trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}42844287mirror.1trial1902026-02-22 22:26:13test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}42844287mirror.1trial1902026-02-22 22:26:13' 2026-02-22T22:26:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:26:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:13' 2026-02-22T22:26:17.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:26:17.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:26:17.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:26:17.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:26:17.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:26:17.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:26:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:18.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:26:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:26:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:26:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:26:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:26:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:26:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:26:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:26:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}} 2026-02-22T22:26:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:13' 2026-02-22T22:26:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:26:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:26:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:26:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:26:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:19.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:26:19.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:26:19.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:19.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:26:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:26:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK2test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:26:13test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:26:13' 2026-02-22T22:26:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:26:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:26:19.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:26:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:26:19.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:26:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:26:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:26:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:26:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:26:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:19.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:26:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:26:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:26:19.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:26:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:19.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:19.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:19.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:19.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:1 mirror test2 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:1 mirror test2 started 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:19.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:1 mirror test2 started 2026-02-22T22:26:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:26:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:26:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:26:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:1 rbd mirror status mirror/test2 2026-02-22T22:26:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:26:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:1 2026-02-22T22:26:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=1 2026-02-22T22:26:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:26:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:26:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test2 2026-02-22T22:26:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:26:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:20.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:26:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:26:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:26:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:26:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:26:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:26:20.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:20.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:20.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:20.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:20.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test2 2026-02-22T22:26:20.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:26:20.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:26:20.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:26:20.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:26:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:26:20.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:26:20.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:26:20.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:20.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T22:26:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:26:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:26:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:26:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:26:20.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:26:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:26:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6249.6 bytes/sec: 24 MiB/s 2026-02-22T22:26:20.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:1 cluster2 mirror test2 2026-02-22T22:26:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:26:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:1 cluster2 mirror test2 2026-02-22T22:26:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:20.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:26:20.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:26:20.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:20.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:20.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:26:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:20.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:20.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:20.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:20.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:20.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:20.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:20.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:26:20.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:20.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:20.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:20.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:20.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:20.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:20.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:20.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:20.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:20.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:20.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:20.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:20.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:20.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:20.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:20.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:20.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:20.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:20.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:20.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:20.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:20.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:20.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:20.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:20.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:20.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:20.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:20.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:20.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:20.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:20.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:20.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:20.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' '' 2026-02-22T22:26:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' = '' 2026-02-22T22:26:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:20.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=3 2026-02-22T22:26:20.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:20.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:20.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=2 2026-02-22T22:26:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:20.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:20.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:26:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:20.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:26:20.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 3 = 2 -a 101 -le 101 2026-02-22T22:26:20.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:20.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:20.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:21.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:21.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:21.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:21.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:21.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:26:21.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:21.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:21.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:21.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:21.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:21.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:21.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:21.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:21.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:26:21.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:21.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:21.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:21.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:21.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:21.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:21.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:21.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:21.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:21.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:21.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:21.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:21.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:21.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:21.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:21.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:21.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:21.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:21.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:21.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:21.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:21.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:21.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:21.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:26:22.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:26:22.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:22.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:22.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:26:22.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:22.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:22.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:22.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:22.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:22.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:22.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:22.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:22.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:22.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:22.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:26:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test2 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test2 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:23.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:23.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:23.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:23.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:23.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test2 2026-02-22T22:26:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test2: 2026-02-22T22:26:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0 2026-02-22T22:26:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}} 2026-02-22T22:26:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.1 on trial190 2026-02-22T22:26:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:13' 2026-02-22T22:26:23.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:26:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:26:23.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:26:23.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}} 2026-02-22T22:26:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.1 on trial190' ']' 2026-02-22T22:26:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.1 on trial190' 2026-02-22T22:26:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.1 on trial190 2026-02-22T22:26:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:13' 2026-02-22T22:26:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test2 up+replaying '2026-02-22 22:26:13' 2026-02-22T22:26:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:26:13' 2026-02-22T22:26:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:23.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING242664269mirror.0trial19019.2.3-689-gf90f4eaefalseOK42844287mirror.1trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}42844287mirror.1trial1902026-02-22 22:26:13test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}42844287mirror.1trial1902026-02-22 22:26:13' 2026-02-22T22:26:23.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:23.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/last_update' 2026-02-22T22:26:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:13' 2026-02-22T22:26:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/state' 2026-02-22T22:26:23.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:26:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:26:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:26:23.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:26:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:26:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test2 down+unknown 2026-02-22T22:26:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:23.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:23.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:23.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:23.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:23.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test2 down+unknown '' '' 2026-02-22T22:26:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test2 2026-02-22T22:26:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test2: 2026-02-22T22:26:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0 2026-02-22T22:26:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:26:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:26:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:26:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:26:24.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:26:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}} 2026-02-22T22:26:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:13' 2026-02-22T22:26:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:26:24.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:26:24.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:26:24.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:26:24.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:24.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:24.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:26:24.917 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:26:24.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:24.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test2 down+unknown '' 2026-02-22T22:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:26:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:24.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK2test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:26:13test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:26:13' 2026-02-22T22:26:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/last_update' 2026-02-22T22:26:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/state' 2026-02-22T22:26:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:26:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:26:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:26:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test2 2026-02-22T22:26:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:26:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test2.export 2026-02-22T22:26:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test2.export 2026-02-22T22:26:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export 2026-02-22T22:26:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test2.export 2026-02-22T22:26:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test2.export 2026-02-22T22:26:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test2 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export 2026-02-22T22:26:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test2 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ release_leader 2026-02-22T22:26:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:26:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror leader release' 2026-02-22T22:26:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror leader release 2026-02-22T22:26:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:25.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:25.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:25.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror leader release 2026-02-22T22:26:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader_released 2026-02-22T22:26:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local i 2026-02-22T22:26:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 1 2026-02-22T22:26:25.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 10 2026-02-22T22:26:25.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in `seq 10` 2026-02-22T22:26:25.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:26:25.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:25.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:25.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:25.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:26:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:26:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:26:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:26:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:26.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:26:26.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:26:26.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:26:26.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:26:26.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:26:26.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:26.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:26:26.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:26:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:26:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=0 2026-02-22T22:26:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 0 = 0 2026-02-22T22:26:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: stop first daemon instance and test replay' 2026-02-22T22:26:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:26:26.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:26:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:26:26 TEST: stop first daemon instance and test replay 2026-02-22T22:26:26.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:26:26 TEST: stop first daemon instance and test replay 2026-02-22T22:26:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:0 2026-02-22T22:26:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:26:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:0 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:26:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:26:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:26:26.590 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:26:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=20578 2026-02-22T22:26:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 20578 ']' 2026-02-22T22:26:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill 20578 2026-02-22T22:26:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:26:26.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:26.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:26:26.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=20578 '$2 == pid {print; exit 1}' 2026-02-22T22:26:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:ubuntu 20578 0.6 0.0 783324 53644 ? Ssl 22:25 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:26:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:26:26.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:27.605 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:26:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=20578 '$2 == pid {print; exit 1}' 2026-02-22T22:26:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:26:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=20578 '$2 == pid {print; exit 1}' 2026-02-22T22:26:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:26:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:27.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:27.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:27.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:27.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:27.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:27.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:27.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:27.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:26:27.654 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:27.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:26:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:26:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:26:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:26:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:26:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image3=test3 2026-02-22T22:26:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test3 2026-02-22T22:26:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:26:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test3 2026-02-22T22:26:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=1 2026-02-22T22:26:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test2 test3 2026-02-22T22:26:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:26:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:26:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:1 mirror test1 2026-02-22T22:26:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:1 mirror test1 started 2026-02-22T22:26:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:26:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:26:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:1 mirror test1 started 2026-02-22T22:26:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:26:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:26:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:26:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:26:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:1 2026-02-22T22:26:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:26:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=1 2026-02-22T22:26:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:26:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:26:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:26:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:26:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:28.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:26:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:26:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:26:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:26:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:26:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:26:28.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:28.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:28.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:28.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T22:26:28.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:26:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:26:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:26:28.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:26:28.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:26:28.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:26:28.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:26:28.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:28.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:26:28.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:26:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:26:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:26:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:26:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:26:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:26:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 8332.95 bytes/sec: 33 MiB/s 2026-02-22T22:26:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:1 cluster2 mirror test1 2026-02-22T22:26:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:26:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:1 cluster2 mirror test1 2026-02-22T22:26:28.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:26:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:26:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:26:29.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:29.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:29.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:29.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:29.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:29.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:29.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:29.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:29.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:29.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:29.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:29.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:29.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:26:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:26:29.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:29.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:29.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:29.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:29.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:29.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:29.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:29.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:29.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:26:29.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:29.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' '!=' '' 2026-02-22T22:26:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' = '' 2026-02-22T22:26:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=5 2026-02-22T22:26:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:29.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:29.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=4 2026-02-22T22:26:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:26:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:29.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:29.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:26:29.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:29.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:29.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:29.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:26:29.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 5 = 4 -a 101 -le 101 2026-02-22T22:26:29.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:29.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:29.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:26:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:26:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:26:29.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:29.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:29.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:29.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:26:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:29.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:29.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:26:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:29.521 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:26:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:29.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:29.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:29.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:29.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:29.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:29.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:29.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:29.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:29.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:29.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:29.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:26:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:29.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' '!=' 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:29.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:29.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:29.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:26:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:30.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:30.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:26:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:30.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:30.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:26:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:30.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:30.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:30.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:30.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:26:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' '!=' 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:30.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:26:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:26:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:26:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:26:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:30.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:30.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:30.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:30.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:30.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:30.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:30.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:26:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:30.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:30.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:30.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:30.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:30.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:26:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:26:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:26:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:31.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:31.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:31.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:31.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:31.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:31.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:31.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:26:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:26:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:26:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:26:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:26:31.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:26:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:26:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:26:31.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:26:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:26:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:26:31.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:26:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:26:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:26:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:26:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:26:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:26:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:26:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:32.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}} 2026-02-22T22:26:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.1 on trial190 2026-02-22T22:26:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:28' 2026-02-22T22:26:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:26:32.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:26:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:26:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}} 2026-02-22T22:26:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.1 on trial190' ']' 2026-02-22T22:26:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.1 on trial190' 2026-02-22T22:26:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.1 on trial190 2026-02-22T22:26:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:28' 2026-02-22T22:26:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:26:28' 2026-02-22T22:26:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:26:28' 2026-02-22T22:26:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING342844287mirror.1trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}42844287mirror.1trial1902026-02-22 22:26:28test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}42844287mirror.1trial1902026-02-22 22:26:28test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42844287mirror.1trial1902026-02-22 22:26:31' 2026-02-22T22:26:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:32.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:26:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:28' 2026-02-22T22:26:32.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:26:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:26:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:26:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:26:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:26:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:26:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:26:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:32.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:26:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:33.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:26:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:26:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:26:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:26:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:26:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:26:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:26:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:26:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}} 2026-02-22T22:26:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:28' 2026-02-22T22:26:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:26:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:26:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:26:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:26:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:26:33.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:26:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:26:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:26:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:33.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK3test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:26:28test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:26:28test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:26:31' 2026-02-22T22:26:33.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:33.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:26:33.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:26:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:26:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:26:33.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:26:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:26:33.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:26:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:26:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:26:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:26:33.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:33.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:26:33.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:26:33.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:26:33.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:33.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:33.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:33.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:26:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:33.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:33.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:26:33.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:1 mirror test2 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:1 mirror test2 started 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:33.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:1 mirror test2 started 2026-02-22T22:26:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:26:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:26:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:26:34.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:1 rbd mirror status mirror/test2 2026-02-22T22:26:34.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:26:34.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:1 2026-02-22T22:26:34.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:34.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:34.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=1 2026-02-22T22:26:34.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:26:34.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:34.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:26:34.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test2 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:34.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:26:34.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:34.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:34.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test2 2026-02-22T22:26:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:26:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:26:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:26:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:26:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:26:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:26:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:26:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T22:26:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:26:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:26:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:26:34.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:26:34.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:26:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:26:34.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6249.78 bytes/sec: 24 MiB/s 2026-02-22T22:26:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:1 cluster2 mirror test2 2026-02-22T22:26:34.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:1 cluster2 mirror test2 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:26:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:34.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:35.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:35.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:35.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:35.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:26:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:35.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:35.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:35.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:35.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:35.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' '!=' '' 2026-02-22T22:26:35.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:35.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' = '' 2026-02-22T22:26:35.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:35.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:35.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:35.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=4 2026-02-22T22:26:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:35.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:35.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=3 2026-02-22T22:26:35.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:35.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:35.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:26:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:35.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:35.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:26:35.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 4 = 3 -a 101 -le 101 2026-02-22T22:26:35.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:35.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:35.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:35.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:26:35.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:35.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:35.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:35.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:35.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:26:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:35.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:35.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:35.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:35.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:35.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:35.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:35.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:35.678 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:35.678 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:35.678 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:35.678 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:35.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:35.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:35.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:35.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:35.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' '!=' 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:26:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:26:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:26:36.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:36.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:36.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:36.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:36.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:36.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:36.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:36.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:36.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:36.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:36.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:36.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:36.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:36.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' '!=' 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:36.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:36.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:26:37.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:26:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:26:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:26:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:37.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:37.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:37.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:37.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:26:37.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:26:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:26:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:37.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:37.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:37.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:37.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:37.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:37.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:37.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:37.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:37.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:37.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:26:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:37.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:26:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:26:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:26:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:26:37.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:26:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:37.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:26:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:26:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:26:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:37.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:26:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test2 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:37.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:37.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test2 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test2 2026-02-22T22:26:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test2: 2026-02-22T22:26:38.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0 2026-02-22T22:26:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}} 2026-02-22T22:26:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.1 on trial190 2026-02-22T22:26:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:28' 2026-02-22T22:26:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:26:38.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:26:38.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:26:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}} 2026-02-22T22:26:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.1 on trial190' ']' 2026-02-22T22:26:38.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.1 on trial190' 2026-02-22T22:26:38.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.1 on trial190 2026-02-22T22:26:38.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:38.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:28' 2026-02-22T22:26:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test2 up+replaying '2026-02-22 22:26:28' 2026-02-22T22:26:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:26:28' 2026-02-22T22:26:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:38.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING342844287mirror.1trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}42844287mirror.1trial1902026-02-22 22:26:28test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}42844287mirror.1trial1902026-02-22 22:26:28test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42844287mirror.1trial1902026-02-22 22:26:31' 2026-02-22T22:26:38.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:38.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/last_update' 2026-02-22T22:26:38.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:28' 2026-02-22T22:26:38.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/state' 2026-02-22T22:26:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:26:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:26:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:26:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:26:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:26:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test2 down+unknown 2026-02-22T22:26:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:39.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test2 down+unknown '' '' 2026-02-22T22:26:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test2 2026-02-22T22:26:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test2: 2026-02-22T22:26:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0 2026-02-22T22:26:39.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:26:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:26:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:26:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:26:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:26:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}} 2026-02-22T22:26:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:28' 2026-02-22T22:26:39.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:26:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:26:39.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:26:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:26:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:39.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:26:39.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:26:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test2 down+unknown '' 2026-02-22T22:26:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:26:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK3test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:26:28test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:26:28test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:26:31' 2026-02-22T22:26:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/last_update' 2026-02-22T22:26:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/state' 2026-02-22T22:26:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:26:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:26:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:26:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test2 2026-02-22T22:26:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:26:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:26:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test2.export 2026-02-22T22:26:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test2.export 2026-02-22T22:26:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export 2026-02-22T22:26:39.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test2.export 2026-02-22T22:26:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test2.export 2026-02-22T22:26:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test2 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export 2026-02-22T22:26:39.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test2 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:39.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:39.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:1 mirror test3 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:1 mirror test3 started 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:26:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:26:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:1 mirror test3 started 2026-02-22T22:26:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:26:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:26:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:26:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:1 rbd mirror status mirror/test3 2026-02-22T22:26:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:26:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:1 2026-02-22T22:26:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=1 2026-02-22T22:26:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:26:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:26:40.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:40.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test3 2026-02-22T22:26:40.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:40.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:26:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:26:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:26:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:26:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test3 2026-02-22T22:26:40.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:26:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:26:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:26:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:26:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:26:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:26:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:26:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test3 100 2026-02-22T22:26:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:26:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:26:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:26:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:26:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:26:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:26:40.949 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6249.84 bytes/sec: 24 MiB/s 2026-02-22T22:26:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:1 cluster2 mirror test3 2026-02-22T22:26:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:1 cluster2 mirror test3 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test3 2026-02-22T22:26:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test3 ']' 2026-02-22T22:26:41.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test3' 2026-02-22T22:26:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test3 2026-02-22T22:26:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test3 2026-02-22T22:26:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:41.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:41.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test3 2026-02-22T22:26:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test3 2026-02-22T22:26:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:41.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test3 '' 2026-02-22T22:26:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test3.status 2026-02-22T22:26:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test3.status 2026-02-22T22:26:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test3 2026-02-22T22:26:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:41.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:41.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test3 2026-02-22T22:26:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test3 '..*' 2026-02-22T22:26:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test3.status 2026-02-22T22:26:41.277 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test3.status 2026-02-22T22:26:41.277 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:41.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:41.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test3 2026-02-22T22:26:41.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:26:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test3 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test3 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:42.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:42.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:42.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:42.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test3 2026-02-22T22:26:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test3: 2026-02-22T22:26:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 5b142ce3-e021-4535-96a0-b1d925ba9b83 2026-02-22T22:26:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.1 on trial190 2026-02-22T22:26:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:31' 2026-02-22T22:26:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:26:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:26:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:26:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.1 on trial190' ']' 2026-02-22T22:26:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.1 on trial190' 2026-02-22T22:26:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.1 on trial190 2026-02-22T22:26:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:31' 2026-02-22T22:26:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test3 up+replaying '2026-02-22 22:26:31' 2026-02-22T22:26:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:26:31' 2026-02-22T22:26:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:42.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING342844287mirror.1trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}42844287mirror.1trial1902026-02-22 22:26:28test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}42844287mirror.1trial1902026-02-22 22:26:28test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42844287mirror.1trial1902026-02-22 22:26:31' 2026-02-22T22:26:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test3'\'']/last_update' 2026-02-22T22:26:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:31' 2026-02-22T22:26:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test3'\'']/state' 2026-02-22T22:26:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:26:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:26:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:26:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:26:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:26:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test3 down+unknown 2026-02-22T22:26:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test3 down+unknown '' '' 2026-02-22T22:26:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:43.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:43.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test3 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test3: 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 5b142ce3-e021-4535-96a0-b1d925ba9b83 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:31' 2026-02-22T22:26:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:26:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:26:43.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:26:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:26:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:26:43.452 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:26:43.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test3 down+unknown '' 2026-02-22T22:26:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:26:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:43.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK3test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:26:28test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:26:28test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:26:31' 2026-02-22T22:26:43.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:43.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test3'\'']/last_update' 2026-02-22T22:26:43.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test3'\'']/state' 2026-02-22T22:26:43.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:26:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:26:43.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:26:43.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:43.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:43.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test3 2026-02-22T22:26:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:26:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test3.export 2026-02-22T22:26:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test3.export 2026-02-22T22:26:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.export 2026-02-22T22:26:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test3.export 2026-02-22T22:26:43.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test3.export 2026-02-22T22:26:43.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:43.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test3.export 2026-02-22T22:26:43.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test3.export 2026-02-22T22:26:43.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test3 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.export 2026-02-22T22:26:44.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:44.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test3 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test3.export 2026-02-22T22:26:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test3.export 2026-02-22T22:26:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test3.export 2026-02-22T22:26:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: start first daemon instance and test replay' 2026-02-22T22:26:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:26:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:26:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:26:44 TEST: start first daemon instance and test replay 2026-02-22T22:26:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:26:44 TEST: start first daemon instance and test replay 2026-02-22T22:26:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:0 2026-02-22T22:26:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:26:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:26:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:26:44.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:26:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:26:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:26:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:26:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:44.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:26:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image4=test4 2026-02-22T22:26:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test4 2026-02-22T22:26:44.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:26:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:44.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test4 2026-02-22T22:26:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test3 test4 2026-02-22T22:26:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:26:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:26:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:1 mirror test3 2026-02-22T22:26:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:1 mirror test3 started 2026-02-22T22:26:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:26:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:26:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:44.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:1 mirror test3 started 2026-02-22T22:26:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:26:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:26:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:26:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:1 rbd mirror status mirror/test3 2026-02-22T22:26:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:26:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:1 2026-02-22T22:26:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=1 2026-02-22T22:26:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:26:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test3 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:26:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test3 2026-02-22T22:26:45.461 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:26:45.462 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:26:45.462 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:26:45.462 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:26:45.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:26:45.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:26:45.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:26:45.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:45.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test3 100 2026-02-22T22:26:45.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:45.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:45.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:45.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:26:45.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:26:45.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:45.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:26:45.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:26:45.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:26:45.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:26:45.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6249.88 bytes/sec: 24 MiB/s 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:1 cluster2 mirror test3 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:1 cluster2 mirror test3 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test3 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test3 ']' 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test3' 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test3 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:45.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:45.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:45.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test3 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test3 2026-02-22T22:26:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test3 2026-02-22T22:26:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test3 '' 2026-02-22T22:26:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test3.status 2026-02-22T22:26:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test3.status 2026-02-22T22:26:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test3 2026-02-22T22:26:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:45.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:45.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test3 2026-02-22T22:26:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test3 '..*' 2026-02-22T22:26:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test3.status 2026-02-22T22:26:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test3.status 2026-02-22T22:26:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test3 2026-02-22T22:26:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:45.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:45.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:45.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:45.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:45.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' '' 2026-02-22T22:26:45.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:45.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' = '' 2026-02-22T22:26:45.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:45.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:45.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:45.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:45.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=3 2026-02-22T22:26:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:45.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=2 2026-02-22T22:26:45.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:45.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:45.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:26:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:45.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:26:45.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 3 = 2 -a 101 -le 101 2026-02-22T22:26:45.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:45.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:45.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test3 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test3 ']' 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test3' 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test3 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:46.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test3 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:46.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test3 2026-02-22T22:26:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test3 2026-02-22T22:26:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test3 '' 2026-02-22T22:26:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:46.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test3.status 2026-02-22T22:26:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test3.status 2026-02-22T22:26:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:46.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test3 2026-02-22T22:26:46.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:46.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test3 2026-02-22T22:26:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test3 '..*' 2026-02-22T22:26:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:46.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test3.status 2026-02-22T22:26:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test3.status 2026-02-22T22:26:46.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:46.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test3 2026-02-22T22:26:46.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:46.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:26:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test3 2026-02-22T22:26:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test3 ']' 2026-02-22T22:26:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test3' 2026-02-22T22:26:46.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test3 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test3 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:46.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:46.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:46.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test3 2026-02-22T22:26:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test3 2026-02-22T22:26:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test3 '' 2026-02-22T22:26:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test3.status 2026-02-22T22:26:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test3.status 2026-02-22T22:26:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test3 2026-02-22T22:26:46.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:46.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:46.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:46.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:46.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:46.749 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:46.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test3 2026-02-22T22:26:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test3 '..*' 2026-02-22T22:26:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:46.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test3.status 2026-02-22T22:26:46.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test3.status 2026-02-22T22:26:46.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:46.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test3 2026-02-22T22:26:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:46.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:46.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:46.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:26:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test3 2026-02-22T22:26:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test3 ']' 2026-02-22T22:26:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test3' 2026-02-22T22:26:47.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test3 2026-02-22T22:26:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:47.594 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:47.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:47.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test3 2026-02-22T22:26:47.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:47.597 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:47.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test3 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test3 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test3 '' 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test3.status 2026-02-22T22:26:47.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test3.status 2026-02-22T22:26:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test3 2026-02-22T22:26:47.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:47.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:47.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:47.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:47.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:47.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:47.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:47.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:47.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test3 2026-02-22T22:26:47.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:47.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:47.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:47.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test3 '..*' 2026-02-22T22:26:47.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test3 2026-02-22T22:26:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test3.status 2026-02-22T22:26:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test3.status 2026-02-22T22:26:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:47.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:47.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test3 2026-02-22T22:26:47.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:26:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.status 2026-02-22T22:26:47.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:47.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:26:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:26:47.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test3 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:47.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test3 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test3 2026-02-22T22:26:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test3: 2026-02-22T22:26:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 5b142ce3-e021-4535-96a0-b1d925ba9b83 2026-02-22T22:26:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.1 on trial190 2026-02-22T22:26:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:31' 2026-02-22T22:26:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:26:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:26:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:26:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.1 on trial190' ']' 2026-02-22T22:26:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.1 on trial190' 2026-02-22T22:26:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.1 on trial190 2026-02-22T22:26:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:31' 2026-02-22T22:26:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test3 up+replaying '2026-02-22 22:26:31' 2026-02-22T22:26:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:26:31' 2026-02-22T22:26:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGWARNINGWARNING442844287mirror.1trial19019.2.3-689-gf90f4eaetrueOK4351mirror.0trial19019.2.3-689-gf90f4eaefalseWARNINGnot reporting statustest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}42844287mirror.1trial1902026-02-22 22:26:28test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}42844287mirror.1trial1902026-02-22 22:26:28test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42844287mirror.1trial1902026-02-22 22:26:31test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42844287mirror.1trial1902026-02-22 22:26:47' 2026-02-22T22:26:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:48.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test3'\'']/last_update' 2026-02-22T22:26:48.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:31' 2026-02-22T22:26:48.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test3'\'']/state' 2026-02-22T22:26:48.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:26:48.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:26:48.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:26:48.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:26:48.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test3 down+unknown 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:49.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test3 down+unknown '' '' 2026-02-22T22:26:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test3 2026-02-22T22:26:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test3: 2026-02-22T22:26:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 5b142ce3-e021-4535-96a0-b1d925ba9b83 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:31' 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:26:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:26:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:26:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:26:49.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:26:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test3 down+unknown '' 2026-02-22T22:26:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:26:49.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:49.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK4test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:26:28test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:26:28test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:26:31test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:26:47' 2026-02-22T22:26:49.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:49.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test3'\'']/last_update' 2026-02-22T22:26:49.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:49.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test3'\'']/state' 2026-02-22T22:26:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:26:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:26:49.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:26:49.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:49.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:49.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test3 2026-02-22T22:26:49.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:49.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test3 2026-02-22T22:26:49.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:26:49.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test3.export 2026-02-22T22:26:49.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test3.export 2026-02-22T22:26:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:49.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.export 2026-02-22T22:26:49.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test3.export 2026-02-22T22:26:49.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test3.export 2026-02-22T22:26:49.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test3.export 2026-02-22T22:26:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test3.export 2026-02-22T22:26:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test3 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.export 2026-02-22T22:26:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:50.075 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test3 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test3.export 2026-02-22T22:26:50.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test3.export 2026-02-22T22:26:50.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test3.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test3.export 2026-02-22T22:26:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:26:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:1 mirror test4 2026-02-22T22:26:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:1 mirror test4 started 2026-02-22T22:26:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:50.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:50.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:26:50.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:26:50.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:50.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:51.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:1 mirror test4 started 2026-02-22T22:26:51.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:51.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:51.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:51.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:26:51.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:26:51.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:26:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:1 rbd mirror status mirror/test4 2026-02-22T22:26:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:26:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:1 2026-02-22T22:26:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=1 2026-02-22T22:26:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:26:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test4 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:51.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:26:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:51.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test4 2026-02-22T22:26:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:26:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:26:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:26:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:26:51.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:26:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:26:51.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:26:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test4 100 2026-02-22T22:26:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:26:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:26:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:26:51.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:26:51.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:26:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:26:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6249.92 bytes/sec: 24 MiB/s 2026-02-22T22:26:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:1 cluster2 mirror test4 2026-02-22T22:26:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:26:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:1 cluster2 mirror test4 2026-02-22T22:26:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:26:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:26:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:26:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test4 2026-02-22T22:26:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test4 ']' 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test4' 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test4 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test4 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:51.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:51.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:51.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:51.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:51.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test4 2026-02-22T22:26:51.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:51.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test4 2026-02-22T22:26:51.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:51.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:51.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:51.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '' 2026-02-22T22:26:51.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:51.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:51.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:51.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:51.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:26:51.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:26:51.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:51.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:51.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:26:51.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:51.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:51.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:51.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:51.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:51.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:26:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:51.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:51.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test4 2026-02-22T22:26:51.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:51.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:51.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:51.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '..*' 2026-02-22T22:26:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:26:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:26:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:26:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:26:51.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:51.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:26:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:26:51.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:51.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:26:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:51.830 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:51.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=1 2026-02-22T22:26:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:51.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:26:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:51.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:51.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=3 2026-02-22T22:26:51.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:26:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:51.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test4 2026-02-22T22:26:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test4 ']' 2026-02-22T22:26:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test4' 2026-02-22T22:26:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test4 2026-02-22T22:26:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test4 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:52.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:52.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:52.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:52.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:52.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:52.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:52.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test4 2026-02-22T22:26:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test4 2026-02-22T22:26:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '' 2026-02-22T22:26:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:26:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:26:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:52.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:26:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:52.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=2, tag_tid=2, entry_tid=34], [object_number=1, tag_tid=2, entry_tid=33], [object_number=0, tag_tid=2, entry_tid=32], [object_number=3, tag_tid=2, entry_tid=31]]], state=connected] 2026-02-22T22:26:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test4 2026-02-22T22:26:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '..*' 2026-02-22T22:26:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:26:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:26:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:52.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:26:52.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=2, tag_tid=2, entry_tid=34], [object_number=1, tag_tid=2, entry_tid=33], [object_number=0, tag_tid=2, entry_tid=32], [object_number=3, tag_tid=2, entry_tid=31]]], state=connected] 2026-02-22T22:26:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=2, tag_tid=2, entry_tid=34' 2026-02-22T22:26:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=2, tag_tid=2, entry_tid=34' 2026-02-22T22:26:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=2, tag_tid=2, entry_tid=34' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=2, tag_tid=2, entry_tid=34' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:26:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=2, tag_tid=2, entry_tid=34' 2026-02-22T22:26:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:26:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=2, tag_tid=2, entry_tid=34' 2026-02-22T22:26:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:26:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=2 2026-02-22T22:26:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:52.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:26:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=2, tag_tid=2, entry_tid=34' 2026-02-22T22:26:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:26:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:26:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=34 2026-02-22T22:26:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 2 -a 101 -le 34 2026-02-22T22:26:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:26:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:26:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:26:52.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test4 2026-02-22T22:26:52.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:52.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:52.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:52.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:26:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test4 ']' 2026-02-22T22:26:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test4' 2026-02-22T22:26:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test4 2026-02-22T22:26:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:26:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:26:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:26:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:52.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:26:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test4 2026-02-22T22:26:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:26:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:26:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:26:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:26:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:52.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test4 2026-02-22T22:26:52.594 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:52.594 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test4 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '' 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:26:52.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:52.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:26:52.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:52.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:52.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:52.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test4 2026-02-22T22:26:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '..*' 2026-02-22T22:26:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:26:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:26:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:26:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:26:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:26:52.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:26:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:26:52.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:26:52.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:26:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:26:52.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:26:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:26:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:26:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test4 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:53.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test4 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:26:53.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:53.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:53.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:53.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:53.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:26:53.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:26:53.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:53.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:53.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test4 2026-02-22T22:26:53.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test4: 2026-02-22T22:26:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 1335d58d-7238-49a4-980e-749ad53dd763 2026-02-22T22:26:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.1 on trial190 2026-02-22T22:26:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:47' 2026-02-22T22:26:53.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:26:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:26:53.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:26:53.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:53.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.1 on trial190' ']' 2026-02-22T22:26:53.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.1 on trial190' 2026-02-22T22:26:53.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.1 on trial190 2026-02-22T22:26:53.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:53.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:47' 2026-02-22T22:26:53.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test4 up+replaying '2026-02-22 22:26:47' 2026-02-22T22:26:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:26:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:26:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:26:47' 2026-02-22T22:26:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:53.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING442844287mirror.1trial19019.2.3-689-gf90f4eaetrueOK43514354mirror.0trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}42844287mirror.1trial1902026-02-22 22:26:28test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}42844287mirror.1trial1902026-02-22 22:26:28test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42844287mirror.1trial1902026-02-22 22:26:31test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42844287mirror.1trial1902026-02-22 22:26:47' 2026-02-22T22:26:53.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:53.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test4'\'']/last_update' 2026-02-22T22:26:53.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:26:47' 2026-02-22T22:26:53.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test4'\'']/state' 2026-02-22T22:26:53.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:26:53.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:26:53.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:26:53.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:26:53.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:53.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:53.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:26:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test4 down+unknown 2026-02-22T22:26:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:26:53.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:26:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test4 down+unknown '' '' 2026-02-22T22:26:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:26:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:26:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:54.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test4 2026-02-22T22:26:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test4: 2026-02-22T22:26:54.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 1335d58d-7238-49a4-980e-749ad53dd763 2026-02-22T22:26:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:26:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:26:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:26:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:26:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:26:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:26:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:26:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:26:47' 2026-02-22T22:26:54.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:26:54.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:26:54.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:26:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:26:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:26:54.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:26:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:26:54.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:26:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test4 down+unknown '' 2026-02-22T22:26:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:26:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:26:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:26:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:26:54.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:26:54.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK4test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:26:28test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:26:28test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:26:31test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:26:47' 2026-02-22T22:26:54.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:26:54.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test4'\'']/last_update' 2026-02-22T22:26:54.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:26:54.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test4'\'']/state' 2026-02-22T22:26:54.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:26:54.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:26:54.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:26:54.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:26:54.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:54.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:54.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test4 2026-02-22T22:26:54.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:54.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:26:54.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:26:54.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test4.export 2026-02-22T22:26:54.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test4.export 2026-02-22T22:26:54.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:54.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.export 2026-02-22T22:26:54.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test4.export 2026-02-22T22:26:54.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test4.export 2026-02-22T22:26:54.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:26:54.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test4.export 2026-02-22T22:26:54.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test4.export 2026-02-22T22:26:54.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test4 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.export 2026-02-22T22:26:54.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:54.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test4 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test4.export 2026-02-22T22:26:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:26:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test4.export 2026-02-22T22:26:55.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test4.export 2026-02-22T22:26:55.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:26:55.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: crash leader and test replay' 2026-02-22T22:26:55.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:26:55.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:26:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:26:55 TEST: crash leader and test replay 2026-02-22T22:26:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:26:55 TEST: crash leader and test replay 2026-02-22T22:26:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:1 -KILL 2026-02-22T22:26:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:26:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig=-KILL 2026-02-22T22:26:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:26:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:1 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:26:55.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:26:55.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:26:55.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:26:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=21524 2026-02-22T22:26:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 21524 ']' 2026-02-22T22:26:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill -KILL 21524 2026-02-22T22:26:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:26:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:26:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:26:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=21524 '$2 == pid {print; exit 1}' 2026-02-22T22:26:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:26:55.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:26:55.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=21524 '$2 == pid {print; exit 1}' 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:55.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:55.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:26:55.277 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:55.277 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:1 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:26:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:26:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:26:55.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image5=test5 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test5 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:26:55.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test5 2026-02-22T22:27:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=0 2026-02-22T22:27:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test4 test5 2026-02-22T22:27:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:27:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:27:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test1 2026-02-22T22:27:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:00.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:27:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:27:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:27:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:00.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:27:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:27:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:27:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:27:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:27:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:27:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:01.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:01.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:01.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:27:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:27:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:27:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:27:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:01.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:27:02.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:02.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:27:02.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:02.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:02.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:02.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:02.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:02.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:02.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:27:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:27:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:27:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:27:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 2 2026-02-22T22:27:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:27:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:04.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:04.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:27:04.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:27:04.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:27:04.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:27:04.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:04.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 4 2026-02-22T22:27:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:08.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:08.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:27:08.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:27:08.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:27:08.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:27:08.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:08.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 8 2026-02-22T22:27:16.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:16.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:27:16.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:16.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:16.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:16.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:16.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:16.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:16.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:16.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:27:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:27:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:27:16.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:27:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 8 2026-02-22T22:27:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:27:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:24.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:24.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:24.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:24.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:24.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:27:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:27:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:27:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:27:24.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 8 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:32.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:32.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:32.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:27:32.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:27:32.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:27:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:27:32.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:27:32.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:27:32.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:27:32.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:27:32.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:32.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:27:32.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:32.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:32.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:32.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:27:32.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:27:32.861 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:27:32.861 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:27:32.861 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:27:32.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:27:32.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:27:32.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:27:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:27:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:27:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:27:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:27:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:27:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:27:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:27:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:27:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:27:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:27:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:27:33.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:27:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:33.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:33.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:33.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:33.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:33.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:27:33.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:33.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:33.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:33.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:27:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:27:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:27:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:33.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:33.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:33.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:33.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:33.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:33.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:33.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:33.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:33.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:27:33.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:33.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:33.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=5, entry_tid=101' '!=' '' 2026-02-22T22:27:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:27:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=5, entry_tid=101' = '' 2026-02-22T22:27:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:27:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=6 2026-02-22T22:27:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:27:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:33.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=5 2026-02-22T22:27:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:27:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:27:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:27:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:27:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 6 = 5 -a 101 -le 101 2026-02-22T22:27:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:27:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:33.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:27:33.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:27:33.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:33.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:33.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:33.506 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:33.506 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:33.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:33.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:33.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:33.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:33.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:33.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:33.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:33.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:33.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:33.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:27:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:27:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:27:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:27:33.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:33.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:33.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:33.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:27:33.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:33.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:33.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:33.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=5, entry_tid=101' '!=' 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:33.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:33.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:27:34.075 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:27:34.075 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:34.075 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:34.075 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:34.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:34.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:27:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:27:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:27:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:34.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:34.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:27:34.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:34.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:34.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:27:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:27:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:34.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:34.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:34.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:34.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:34.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:34.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:34.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:27:34.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=5, entry_tid=101' '!=' 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:27:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:27:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:27:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:27:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:27:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:35.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:27:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:35.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:35.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:35.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:27:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:27:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:27:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:27:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:27:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:27:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:35.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:27:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:27:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:27:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:27:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:27:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:27:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:27:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T22:27:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:27:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:26' 2026-02-22T22:27:36.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:27:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:27:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:27:36.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T22:27:36.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:27:36.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:27:36.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:27:36.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:27:36.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:27:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:26' 2026-02-22T22:27:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:27:26' 2026-02-22T22:27:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:36.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:36.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:36.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:36.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:27:26' 2026-02-22T22:27:36.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:36.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:36.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:27:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING542844287mirror.1trial19019.2.3-689-gf90f4eaetrueOK43514354mirror.0trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43514354mirror.0trial1902026-02-22 22:27:26test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}43514354mirror.0trial1902026-02-22 22:27:26test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:26test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:27:26test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:25' 2026-02-22T22:27:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:27:36.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:27:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:26' 2026-02-22T22:27:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:27:36.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:27:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:27:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:27:36.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:27:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:27:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:27:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:36.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:27:36.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:27:36.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:36.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:27:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:27:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:27:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:27:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:27:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:27:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:27:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:27:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:27:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:27:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:27:37.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:27:37.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T22:27:37.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:26' 2026-02-22T22:27:37.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:27:37.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:27:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:27:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:27:37.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:27:37.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:27:37.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:27:37.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:27:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:27:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:27:37.530 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:27:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:27:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:27:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:27:37.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK5test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:27:26test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:27:26test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:26test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:27:26test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:25' 2026-02-22T22:27:37.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:27:37.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:27:37.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:27:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:27:37.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:27:37.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:27:37.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:27:37.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:27:37.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:37.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:27:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:27:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:27:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:27:37.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:27:37.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:27:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:27:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:27:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:27:37.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:27:37.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:27:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:27:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:27:37.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:27:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:27:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:27:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test4 2026-02-22T22:27:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test4 started 2026-02-22T22:27:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:27:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:27:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test4 started 2026-02-22T22:27:38.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:38.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:38.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:38.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:27:38.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:27:38.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:27:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test4 2026-02-22T22:27:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:27:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:27:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test4 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:38.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:38.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test4 2026-02-22T22:27:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:27:39.032 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:27:39.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:27:39.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:27:39.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:27:39.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test4 100 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:27:39.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:27:39.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:27:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:27:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:27:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test4 2026-02-22T22:27:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:27:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:39.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:27:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test4 2026-02-22T22:27:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:27:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:27:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:27:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:27:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:27:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test4 2026-02-22T22:27:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test4 ']' 2026-02-22T22:27:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test4' 2026-02-22T22:27:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:27:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:27:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test4 2026-02-22T22:27:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test4 2026-02-22T22:27:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '' 2026-02-22T22:27:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:27:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:27:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:27:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:39.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:39.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:39.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:27:39.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:27:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test4 2026-02-22T22:27:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '..*' 2026-02-22T22:27:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:27:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:27:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:27:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:39.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:39.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:39.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:27:39.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' '' 2026-02-22T22:27:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:27:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' = '' 2026-02-22T22:27:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:27:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:27:39.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:39.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=3 2026-02-22T22:27:39.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:39.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:27:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:39.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=2 2026-02-22T22:27:39.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:27:39.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:27:39.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:39.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:27:39.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:27:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:27:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 3 = 2 -a 101 -le 101 2026-02-22T22:27:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:27:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:27:39.654 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test4 2026-02-22T22:27:39.654 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test4 ']' 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test4' 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:39.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:27:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:39.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:39.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:39.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:39.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test4 2026-02-22T22:27:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test4 2026-02-22T22:27:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '' 2026-02-22T22:27:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:27:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:27:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:27:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:39.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:39.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:27:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:39.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:27:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test4 2026-02-22T22:27:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:39.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '..*' 2026-02-22T22:27:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:27:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:27:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:27:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:39.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:39.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:39.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:27:39.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:39.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:39.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:39.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:39.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:39.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:39.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:27:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test4 2026-02-22T22:27:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:40.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test4 ']' 2026-02-22T22:27:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test4' 2026-02-22T22:27:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:27:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:40.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:40.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:27:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:40.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:40.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:40.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:40.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test4 2026-02-22T22:27:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test4 2026-02-22T22:27:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '' 2026-02-22T22:27:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:27:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:27:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:27:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:27:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:40.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:27:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test4 2026-02-22T22:27:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '..*' 2026-02-22T22:27:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:27:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:27:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:27:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:27:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=101' '!=' 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:27:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test4 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test4 ']' 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test4' 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:41.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:41.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test4 2026-02-22T22:27:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test4 2026-02-22T22:27:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '' 2026-02-22T22:27:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:27:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:27:41.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:41.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:41.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:27:41.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:27:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:27:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:27:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test4 2026-02-22T22:27:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test4 '..*' 2026-02-22T22:27:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test4 2026-02-22T22:27:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:27:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:27:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:41.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:27:41.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:27:41.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:27:41.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.status 2026-02-22T22:27:41.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:27:41.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=3, entry_tid=101' -a 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:27:41.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:41.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:27:41.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test4 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:27:41.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:27:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:27:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test4 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:27:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:27:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:27:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test4 2026-02-22T22:27:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test4: 2026-02-22T22:27:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 1335d58d-7238-49a4-980e-749ad53dd763 2026-02-22T22:27:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:27:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}} 2026-02-22T22:27:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:27:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:26' 2026-02-22T22:27:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:27:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:27:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:27:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}} 2026-02-22T22:27:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:27:42.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:27:42.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:27:42.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:27:42.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:27:42.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:26' 2026-02-22T22:27:42.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test4 up+replaying '2026-02-22 22:27:26' 2026-02-22T22:27:42.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:42.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:42.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:42.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:42.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:27:26' 2026-02-22T22:27:42.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:42.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:42.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:27:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING542844287mirror.1trial19019.2.3-689-gf90f4eaetrueOK43514354mirror.0trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43514354mirror.0trial1902026-02-22 22:27:26test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}43514354mirror.0trial1902026-02-22 22:27:26test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:26test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:27:26test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:25' 2026-02-22T22:27:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:27:42.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test4'\'']/last_update' 2026-02-22T22:27:42.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:26' 2026-02-22T22:27:42.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test4'\'']/state' 2026-02-22T22:27:42.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:27:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:27:42.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:27:42.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:27:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:27:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test4 down+unknown 2026-02-22T22:27:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:27:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:27:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test4 down+unknown '' '' 2026-02-22T22:27:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:27:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:27:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test4 2026-02-22T22:27:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test4: 2026-02-22T22:27:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 1335d58d-7238-49a4-980e-749ad53dd763 2026-02-22T22:27:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:27:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:27:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:27:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:27:43.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:27:43.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:27:43.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}} 2026-02-22T22:27:43.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:26' 2026-02-22T22:27:43.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:27:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:27:43.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:27:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:27:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:27:43.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:27:43.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:27:43.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:27:43.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:27:43.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:27:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:27:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test4 down+unknown '' 2026-02-22T22:27:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:27:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:43.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:27:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK5test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:27:26test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:27:26test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:26test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:27:26test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:25' 2026-02-22T22:27:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:27:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test4'\'']/last_update' 2026-02-22T22:27:43.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:27:43.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test4'\'']/state' 2026-02-22T22:27:43.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:27:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:27:43.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:27:43.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test4 2026-02-22T22:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test4 2026-02-22T22:27:43.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test4.export 2026-02-22T22:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test4.export 2026-02-22T22:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.export 2026-02-22T22:27:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test4.export 2026-02-22T22:27:43.530 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test4.export 2026-02-22T22:27:43.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:43.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test4.export 2026-02-22T22:27:43.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test4.export 2026-02-22T22:27:43.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test4 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.export 2026-02-22T22:27:43.654 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:27:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test4 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test4.export 2026-02-22T22:27:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:27:43.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test4.export 2026-02-22T22:27:43.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test4.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test4.export 2026-02-22T22:27:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:27:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test5 2026-02-22T22:27:43.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test5 started 2026-02-22T22:27:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:27:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:27:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:43.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:44.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test5 started 2026-02-22T22:27:44.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:27:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:27:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:27:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test5 2026-02-22T22:27:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:27:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:27:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:27:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:27:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:27:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test5 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:44.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test5 2026-02-22T22:27:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:27:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:27:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:27:44.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:27:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:27:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:27:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:27:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test5 100 2026-02-22T22:27:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:27:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:27:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:27:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:27:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:27:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:27:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:27:45.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:27:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test5 2026-02-22T22:27:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:27:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:27:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test5 2026-02-22T22:27:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:27:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:27:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:27:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:27:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:27:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test5 2026-02-22T22:27:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test5 ']' 2026-02-22T22:27:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test5' 2026-02-22T22:27:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test5 2026-02-22T22:27:45.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test5 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:45.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:45.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:45.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:45.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test5 2026-02-22T22:27:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test5 2026-02-22T22:27:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test5 '' 2026-02-22T22:27:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test5.status 2026-02-22T22:27:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test5.status 2026-02-22T22:27:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:45.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test5 2026-02-22T22:27:45.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:27:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test5 2026-02-22T22:27:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test5 '..*' 2026-02-22T22:27:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test5.status 2026-02-22T22:27:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test5.status 2026-02-22T22:27:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test5 2026-02-22T22:27:45.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:45.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:27:45.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:27:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:27:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:27:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:27:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:27:45.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:45.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:27:45.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=1 2026-02-22T22:27:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:27:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:27:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:45.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:27:45.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:45.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=3 2026-02-22T22:27:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:27:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:27:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:27:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test5 2026-02-22T22:27:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test5 ']' 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test5' 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test5 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test5 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:45.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:45.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test5 2026-02-22T22:27:45.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test5 2026-02-22T22:27:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test5 '' 2026-02-22T22:27:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test5.status 2026-02-22T22:27:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test5.status 2026-02-22T22:27:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test5 2026-02-22T22:27:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:27:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.749 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test5 2026-02-22T22:27:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test5 '..*' 2026-02-22T22:27:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:45.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test5.status 2026-02-22T22:27:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test5.status 2026-02-22T22:27:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:45.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test5 2026-02-22T22:27:45.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:45.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:45.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:45.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:45.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:27:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:27:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test5 2026-02-22T22:27:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test5 ']' 2026-02-22T22:27:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test5' 2026-02-22T22:27:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test5 2026-02-22T22:27:46.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test5 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:46.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:46.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:46.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:46.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:46.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:46.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test5 2026-02-22T22:27:46.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:46.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test5 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test5 '' 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test5.status 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test5.status 2026-02-22T22:27:46.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test5 2026-02-22T22:27:46.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:46.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:46.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:46.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:46.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:46.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:46.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:46.315 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:46.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test5 2026-02-22T22:27:46.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:46.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:46.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:46.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test5 '..*' 2026-02-22T22:27:46.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:46.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:46.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test5 2026-02-22T22:27:46.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:46.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test5.status 2026-02-22T22:27:46.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test5.status 2026-02-22T22:27:46.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:46.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:46.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test5 2026-02-22T22:27:46.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:46.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:46.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.status 2026-02-22T22:27:46.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:46.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:46.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:46.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:27:46.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test5 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:27:46.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:46.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:46.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:46.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:46.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:27:46.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:27:46.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:46.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:47.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test5 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:27:47.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:47.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:47.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:47.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:47.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:27:47.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:27:47.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:47.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:47.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test5 2026-02-22T22:27:47.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test5: 2026-02-22T22:27:47.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 27d435c7-b11b-4cdb-9cab-ccb4d4e8032e 2026-02-22T22:27:47.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:27:47.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:27:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:27:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:44' 2026-02-22T22:27:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:27:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:27:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:27:47.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:27:47.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:27:47.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:27:47.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:27:47.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:27:47.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:27:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:44' 2026-02-22T22:27:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test5 up+replaying '2026-02-22 22:27:44' 2026-02-22T22:27:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:27:44' 2026-02-22T22:27:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:47.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:27:47.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING542844287mirror.1trial19019.2.3-689-gf90f4eaetrueOK43514354mirror.0trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}43514354mirror.0trial1902026-02-22 22:27:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}43514354mirror.0trial1902026-02-22 22:27:44test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:44test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":27603.2,"entries_behind_primary":0,"entries_per_second":6.8,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:44' 2026-02-22T22:27:47.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:27:47.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test5'\'']/last_update' 2026-02-22T22:27:47.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:44' 2026-02-22T22:27:47.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test5'\'']/state' 2026-02-22T22:27:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:27:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:27:47.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:27:47.454 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:27:47.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:47.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:27:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test5 down+unknown 2026-02-22T22:27:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:27:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:27:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:47.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:48.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test5 down+unknown '' '' 2026-02-22T22:27:48.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:48.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:48.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:48.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:48.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:27:48.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:27:48.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:48.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:48.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test5 2026-02-22T22:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test5: 2026-02-22T22:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 27d435c7-b11b-4cdb-9cab-ccb4d4e8032e 2026-02-22T22:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:27:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:27:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:27:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:44' 2026-02-22T22:27:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:27:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:27:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:27:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:27:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:27:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:27:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:27:48.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:27:48.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:27:48.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:27:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test5 down+unknown '' 2026-02-22T22:27:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:27:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:27:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK5test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:27:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:27:44test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:44test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":27603.2,"entries_behind_primary":0,"entries_per_second":6.8,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:44' 2026-02-22T22:27:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:27:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test5'\'']/last_update' 2026-02-22T22:27:48.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:27:48.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test5'\'']/state' 2026-02-22T22:27:48.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:27:48.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:27:48.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:27:48.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:27:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test5 2026-02-22T22:27:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test5 2026-02-22T22:27:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:27:48.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test5.export 2026-02-22T22:27:48.545 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test5.export 2026-02-22T22:27:48.545 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:48.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.export 2026-02-22T22:27:48.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test5.export 2026-02-22T22:27:48.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test5.export 2026-02-22T22:27:48.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test5.export 2026-02-22T22:27:48.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test5.export 2026-02-22T22:27:48.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test5 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.export 2026-02-22T22:27:48.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:27:48.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test5 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test5.export 2026-02-22T22:27:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:27:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test5.export 2026-02-22T22:27:48.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test5.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test5.export 2026-02-22T22:27:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: start crashed leader and test replay' 2026-02-22T22:27:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:27:48.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:27:48.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:27:48 TEST: start crashed leader and test replay 2026-02-22T22:27:48.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:27:48 TEST: start crashed leader and test replay 2026-02-22T22:27:48.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:1 2026-02-22T22:27:48.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:27:48.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:48.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:27:48.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:27:48.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:27:48.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image6=test6 2026-02-22T22:27:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test6 2026-02-22T22:27:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:27:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:27:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test6 2026-02-22T22:27:48.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test6 2026-02-22T22:27:48.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test1 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:27:48.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:27:48.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:48.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:49.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:27:49.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:27:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:27:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:27:49.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:27:49.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:27:49.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:27:49.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:49.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:49.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:27:49.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:27:49.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:49.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:27:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:27:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:49.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:49.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:49.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:49.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:49.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:49.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:49.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:27:50.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:27:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:27:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:27:50.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:27:50.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:27:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:27:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:27:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:27:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:50.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:27:50.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:27:50.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:27:50.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:27:50.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:27:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:27:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:27:50.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 8333.48 bytes/sec: 33 MiB/s 2026-02-22T22:27:50.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:27:50.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:27:50.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:27:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:27:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:50.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:27:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:27:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:27:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:27:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:50.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:50.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:50.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:50.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:50.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:50.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:50.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:50.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:27:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:50.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:50.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:50.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:50.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:27:50.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:27:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:27:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:50.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:50.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:50.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:50.445 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.445 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:50.445 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:50.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:50.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:50.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:27:50.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=7, entry_tid=101' -a 'object_number=1, tag_tid=7, entry_tid=101' = 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=6, entry_tid=101' '!=' '' 2026-02-22T22:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=6, entry_tid=101' = '' 2026-02-22T22:27:50.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:50.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:27:50.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:27:50.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=7 2026-02-22T22:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:27:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=6 2026-02-22T22:27:50.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:27:50.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:27:50.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:50.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:27:50.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 7 = 6 -a 101 -le 101 2026-02-22T22:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:27:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:27:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:27:50.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:27:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:50.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:27:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:50.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:50.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:27:50.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:50.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:27:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:27:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:27:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:50.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:50.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:50.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:50.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:50.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:50.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:50.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:50.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:50.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:27:50.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:50.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=7, entry_tid=101' -a 'object_number=1, tag_tid=7, entry_tid=101' = 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=6, entry_tid=101' '!=' 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:27:51.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:51.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:51.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:51.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:51.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:51.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:51.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:51.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:51.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:27:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:27:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:27:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:51.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:51.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:51.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:27:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:51.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:51.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:27:51.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:27:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:27:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:51.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:51.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:51.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:51.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:51.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:51.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:51.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:51.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:51.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:51.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:27:51.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:27:51.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:51.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:51.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=7, entry_tid=101' -a 'object_number=1, tag_tid=7, entry_tid=101' = 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:51.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=6, entry_tid=101' '!=' 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:27:51.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:51.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:27:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:27:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:52.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:27:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:52.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:27:52.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:52.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:52.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:27:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:52.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:52.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:27:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:27:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:27:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:27:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:27:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:27:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:27:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:27:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:27:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:52.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:27:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:27:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:27:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:27:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=7, entry_tid=101' -a 'object_number=1, tag_tid=7, entry_tid=101' = 'object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:27:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:27:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:27:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:27:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:27:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:27:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:27:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:27:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:27:53.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:27:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:27:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:27:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}} 2026-02-22T22:27:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:27:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:44' 2026-02-22T22:27:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:27:53.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:27:53.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:27:53.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}} 2026-02-22T22:27:53.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:27:53.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:27:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:27:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:27:53.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:27:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:44' 2026-02-22T22:27:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:27:44' 2026-02-22T22:27:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:53.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:53.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:53.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:27:44' 2026-02-22T22:27:53.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:53.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:53.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:27:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGWARNINGWARNING642844287mirror.1trial19019.2.3-689-gf90f4eaetrueOK43514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK4404mirror.1trial19019.2.3-689-gf90f4eaefalseWARNINGnot reporting statustest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}43514354mirror.0trial1902026-02-22 22:27:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}43514354mirror.0trial1902026-02-22 22:27:44test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:44test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":27603.2,"entries_behind_primary":0,"entries_per_second":6.8,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:52' 2026-02-22T22:27:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:27:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:27:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:44' 2026-02-22T22:27:53.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:27:53.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:27:53.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:27:53.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:27:53.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:27:53.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:53.449 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:27:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:27:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:27:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:27:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:53.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:27:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:27:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:27:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:27:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:27:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:27:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:27:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:27:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:27:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:27:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:27:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:27:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}} 2026-02-22T22:27:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:44' 2026-02-22T22:27:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:27:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:27:54.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:27:54.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:27:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:27:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:27:54.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:27:54.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:27:54.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:27:54.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:27:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:27:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:27:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:27:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:27:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK6test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:27:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:27:44test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:44test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":27603.2,"entries_behind_primary":0,"entries_per_second":6.8,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:52' 2026-02-22T22:27:54.530 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:27:54.530 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:27:54.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:27:54.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:27:54.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:27:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:27:54.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:27:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:27:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:27:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:27:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:27:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:27:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:27:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:27:54.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:27:54.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:27:54.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:27:54.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:27:54.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:27:54.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:27:54.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:27:54.794 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:27:54.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:27:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:27:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test6 2026-02-22T22:27:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:54.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test6 started 2026-02-22T22:27:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:27:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:27:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:54.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:55.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test6 started 2026-02-22T22:27:55.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:55.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:55.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:55.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:27:55.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:27:55.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:27:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test6 2026-02-22T22:27:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:27:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:27:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:27:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:27:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:55.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:27:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test6 2026-02-22T22:27:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:27:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:27:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:55.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:55.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:55.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:55.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test6 2026-02-22T22:27:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:27:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:27:55.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:27:55.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test6 100 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:27:55.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:27:55.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:27:55.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test6 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:27:56.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:27:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:27:56.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:27:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test6 2026-02-22T22:27:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:27:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:27:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test6 2026-02-22T22:27:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:27:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:56.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:27:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:27:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:27:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:27:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test6 2026-02-22T22:27:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test6 ']' 2026-02-22T22:27:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test6' 2026-02-22T22:27:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test6 2026-02-22T22:27:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test6 2026-02-22T22:27:56.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:56.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:56.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:56.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:56.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test6 2026-02-22T22:27:56.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:56.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test6 2026-02-22T22:27:56.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:56.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:56.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:56.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test6 '' 2026-02-22T22:27:56.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:56.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:56.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:56.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:56.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test6.status 2026-02-22T22:27:56.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test6.status 2026-02-22T22:27:56.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:56.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test6 2026-02-22T22:27:56.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:27:56.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test6 2026-02-22T22:27:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test6 '..*' 2026-02-22T22:27:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:56.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test6.status 2026-02-22T22:27:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test6.status 2026-02-22T22:27:56.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test6 2026-02-22T22:27:56.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:56.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:56.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:56.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:56.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:27:56.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:56.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:27:56.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:56.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:27:56.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:56.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=1 2026-02-22T22:27:56.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:56.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:27:56.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:56.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:27:56.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:56.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:27:56.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:27:56.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=3 2026-02-22T22:27:56.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:27:56.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:27:56.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:56.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:27:56.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test6 2026-02-22T22:27:56.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test6 ']' 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test6' 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test6 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test6 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:56.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:56.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:56.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:56.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test6 2026-02-22T22:27:56.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:56.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test6 2026-02-22T22:27:56.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:56.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:56.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:56.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test6 '' 2026-02-22T22:27:56.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:56.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:56.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:56.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:56.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test6.status 2026-02-22T22:27:56.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test6.status 2026-02-22T22:27:56.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:56.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test6 2026-02-22T22:27:56.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:56.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:56.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:56.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:27:56.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:56.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test6 2026-02-22T22:27:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test6 '..*' 2026-02-22T22:27:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:56.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test6.status 2026-02-22T22:27:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test6.status 2026-02-22T22:27:56.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test6 2026-02-22T22:27:56.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:56.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:27:56.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:27:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:27:56.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:27:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test6 2026-02-22T22:27:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:27:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test6 ']' 2026-02-22T22:27:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test6' 2026-02-22T22:27:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test6 2026-02-22T22:27:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:27:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:27:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test6 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:57.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:27:57.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:27:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:27:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:27:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:27:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:27:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:27:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:27:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:27:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:27:57.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:27:57.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:57.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:57.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:27:57.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test6 2026-02-22T22:27:57.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test6 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test6 '' 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test6.status 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test6.status 2026-02-22T22:27:57.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test6 2026-02-22T22:27:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:57.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:57.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:57.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:57.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:57.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:57.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:57.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test6 2026-02-22T22:27:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test6 '..*' 2026-02-22T22:27:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:27:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:27:57.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test6 2026-02-22T22:27:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:27:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test6.status 2026-02-22T22:27:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test6.status 2026-02-22T22:27:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:57.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:57.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test6 2026-02-22T22:27:57.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:57.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:27:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:27:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:27:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:57.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:27:57.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.status 2026-02-22T22:27:57.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:57.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:27:57.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:57.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:27:57.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test6 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:27:57.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:57.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:57.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:57.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:57.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:27:57.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:27:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test6 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:27:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:27:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:27:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test6 2026-02-22T22:27:58.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test6: 2026-02-22T22:27:58.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 9f2c857c-e2fe-4ac6-9969-3ab6676435e7 2026-02-22T22:27:58.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:27:58.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:27:58.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:27:58.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:52' 2026-02-22T22:27:58.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:27:58.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:27:58.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:27:58.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:27:58.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:27:58.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:27:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:27:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:27:58.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:27:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:52' 2026-02-22T22:27:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test6 up+replaying '2026-02-22 22:27:52' 2026-02-22T22:27:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:27:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:27:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:27:52' 2026-02-22T22:27:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:27:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING643514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44044407mirror.1trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}43514354mirror.0trial1902026-02-22 22:27:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}43514354mirror.0trial1902026-02-22 22:27:44test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:44test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":27603.2,"entries_behind_primary":0,"entries_per_second":6.8,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:52' 2026-02-22T22:27:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:27:58.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test6'\'']/last_update' 2026-02-22T22:27:58.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:52' 2026-02-22T22:27:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test6'\'']/state' 2026-02-22T22:27:58.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:27:58.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:27:58.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:27:58.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:27:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:27:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test6 down+unknown 2026-02-22T22:27:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:58.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:58.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:58.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:27:58.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:27:58.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:58.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:27:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test6 down+unknown '' '' 2026-02-22T22:27:59.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:27:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:27:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test6 2026-02-22T22:27:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test6: 2026-02-22T22:27:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 9f2c857c-e2fe-4ac6-9969-3ab6676435e7 2026-02-22T22:27:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:27:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:27:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:27:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:27:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:27:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:27:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:27:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:52' 2026-02-22T22:27:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:27:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:27:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:27:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:27:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:27:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:27:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:27:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:27:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:27:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:27:59.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:27:59.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test6 down+unknown '' 2026-02-22T22:27:59.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:59.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:59.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:59.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:27:59.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:27:59.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:27:59.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:27:59.482 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:27:59.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK6test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:27:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:27:44test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:44test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":27603.2,"entries_behind_primary":0,"entries_per_second":6.8,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:52' 2026-02-22T22:27:59.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:27:59.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test6'\'']/last_update' 2026-02-22T22:27:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:27:59.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test6'\'']/state' 2026-02-22T22:27:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:27:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:27:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:27:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:27:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test6 2026-02-22T22:27:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test6 2026-02-22T22:27:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:27:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test6.export 2026-02-22T22:27:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test6.export 2026-02-22T22:27:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.export 2026-02-22T22:27:59.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test6.export 2026-02-22T22:27:59.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test6.export 2026-02-22T22:27:59.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:27:59.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test6.export 2026-02-22T22:27:59.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test6.export 2026-02-22T22:27:59.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test6 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.export 2026-02-22T22:27:59.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:27:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test6 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test6.export 2026-02-22T22:27:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:27:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test6.export 2026-02-22T22:27:59.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test6.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test6.export 2026-02-22T22:27:59.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:27:59.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: start yet another daemon instance and test replay' 2026-02-22T22:27:59.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:27:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:27:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:27:59 TEST: start yet another daemon instance and test replay 2026-02-22T22:27:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:27:59 TEST: start yet another daemon instance and test replay 2026-02-22T22:27:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:2 2026-02-22T22:27:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:27:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:27:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:27:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:27:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:27:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:27:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:27:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:27:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:27:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:27:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:27:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:27:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:27:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:27:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:27:59.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image7=test7 2026-02-22T22:27:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test7 2026-02-22T22:27:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:27:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:27:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:27:59.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:27:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:27:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test7 2026-02-22T22:27:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test7 2026-02-22T22:27:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:27:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:27:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test1 2026-02-22T22:27:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:27:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:27:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:27:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:27:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:27:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:27:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:27:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:28:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:28:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:28:00.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:28:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:28:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:28:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:28:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:28:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:28:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:00.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:28:00.949 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:00.949 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:00.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:28:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:28:00.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:00.960 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:00.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:00.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:28:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:28:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:28:01.019 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:28:01.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:28:01.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:28:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:28:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:28:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:28:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:01.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:28:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:28:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:28:01.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:28:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:28:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:28:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:28:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:28:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:28:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:28:01.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:28:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:28:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:01.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:01.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:01.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:01.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:01.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:01.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:01.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:01.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:28:01.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:01.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:28:01.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:01.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=8, entry_tid=101' -a 'object_number=1, tag_tid=8, entry_tid=101' = 'object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:01.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=7, entry_tid=101' '!=' '' 2026-02-22T22:28:01.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:01.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=7, entry_tid=101' = '' 2026-02-22T22:28:01.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:01.434 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:01.434 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=8 2026-02-22T22:28:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:01.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:01.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=7 2026-02-22T22:28:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:28:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:28:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 8 = 7 -a 101 -le 101 2026-02-22T22:28:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:01.441 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:01.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:01.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:01.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:01.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:01.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:01.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:01.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:01.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:01.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:01.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:01.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:01.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:01.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:28:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:01.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:01.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:01.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:01.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:01.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:01.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:01.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:28:01.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:01.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:01.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=8, entry_tid=101' -a 'object_number=1, tag_tid=8, entry_tid=101' = 'object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:01.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=7, entry_tid=101' '!=' 'object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:01.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:01.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:28:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:02.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:02.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:02.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:02.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:02.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:02.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:02.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:02.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:28:02.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=7, entry_tid=101], [object_number=0, tag_tid=7, entry_tid=100], [object_number=3, tag_tid=7, entry_tid=99], [object_number=2, tag_tid=7, entry_tid=98]]], state=connected] 2026-02-22T22:28:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=8, entry_tid=101' -a 'object_number=1, tag_tid=8, entry_tid=101' = 'object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=7, entry_tid=101' '!=' 'object_number=1, tag_tid=7, entry_tid=101' 2026-02-22T22:28:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:28:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:03.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:03.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:03.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:03.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:03.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:03.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:03.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:03.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:03.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:03.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:03.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:03.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=8, entry_tid=101' -a 'object_number=1, tag_tid=8, entry_tid=101' = 'object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:03.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:03.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:28:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:28:04.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:28:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:28:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}} 2026-02-22T22:28:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:28:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:44' 2026-02-22T22:28:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:28:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:28:04.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}} 2026-02-22T22:28:04.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:28:04.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:28:04.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:28:04.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:04.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:44' 2026-02-22T22:28:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:27:44' 2026-02-22T22:28:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:27:44' 2026-02-22T22:28:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGWARNINGWARNING743514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44044407mirror.1trial19019.2.3-689-gf90f4eaefalseOK4431mirror.2trial19019.2.3-689-gf90f4eaefalseWARNINGnot reporting statustest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}43514354mirror.0trial1902026-02-22 22:27:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}43514354mirror.0trial1902026-02-22 22:27:44test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:44test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":27603.2,"entries_behind_primary":0,"entries_per_second":6.8,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:52test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:28:03' 2026-02-22T22:28:04.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:04.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:28:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:27:44' 2026-02-22T22:28:04.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:28:04.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:28:04.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:28:04.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:28:04.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:28:04.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:05.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:28:05.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:05.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:05.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:05.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:28:05.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:28:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:28:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:28:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:28:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:28:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:28:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:28:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}} 2026-02-22T22:28:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:27:44' 2026-02-22T22:28:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:28:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:28:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:28:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:28:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:28:05.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:28:05.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:05.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:28:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:28:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:05.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK7test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:27:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:27:44test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:44test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":27603.2,"entries_behind_primary":0,"entries_per_second":6.8,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:52test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:28:03' 2026-02-22T22:28:05.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:05.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:28:05.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:28:05.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:28:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:28:05.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:28:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:28:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:28:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:28:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:28:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:28:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:28:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:28:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:05.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:28:05.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:05.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:05.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:05.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:05.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:28:05.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test7 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test7 started 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test7 started 2026-02-22T22:28:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:28:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:28:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:28:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test7 2026-02-22T22:28:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:28:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:28:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:28:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:28:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:28:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test7 2026-02-22T22:28:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:28:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:28:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:28:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:06.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test7 2026-02-22T22:28:06.948 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:28:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:28:06.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:28:06.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:28:06.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:28:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:28:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:28:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test7 100 2026-02-22T22:28:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:28:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:28:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:28:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test7 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:28:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:28:07.003 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:28:07.003 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:28:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test7 2026-02-22T22:28:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test7 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:07.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test7 2026-02-22T22:28:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test7 ']' 2026-02-22T22:28:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test7' 2026-02-22T22:28:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test7 2026-02-22T22:28:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:07.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test7 2026-02-22T22:28:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:07.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:07.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test7 2026-02-22T22:28:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test7 2026-02-22T22:28:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test7 '' 2026-02-22T22:28:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test7.status 2026-02-22T22:28:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test7.status 2026-02-22T22:28:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test7 2026-02-22T22:28:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:28:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test7 2026-02-22T22:28:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test7 '..*' 2026-02-22T22:28:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:07.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:07.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:07.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test7.status 2026-02-22T22:28:07.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test7.status 2026-02-22T22:28:07.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test7 2026-02-22T22:28:07.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:07.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:07.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:28:07.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:28:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:28:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:07.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:07.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:07.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:07.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:28:07.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:07.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:07.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:07.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=1 2026-02-22T22:28:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:07.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:07.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:28:07.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:07.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:07.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:07.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=3 2026-02-22T22:28:07.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:28:07.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:07.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:07.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test7 2026-02-22T22:28:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test7 ']' 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test7' 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test7 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test7 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:07.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test7 2026-02-22T22:28:07.658 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:07.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test7 2026-02-22T22:28:07.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:07.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:07.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:07.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test7 '' 2026-02-22T22:28:07.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:07.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:07.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:07.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:07.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test7.status 2026-02-22T22:28:07.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test7.status 2026-02-22T22:28:07.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:07.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test7 2026-02-22T22:28:07.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:07.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:07.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:07.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:07.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:28:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test7 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test7 '..*' 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test7.status 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test7.status 2026-02-22T22:28:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test7 2026-02-22T22:28:07.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:28:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:28:08.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test7 2026-02-22T22:28:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test7 ']' 2026-02-22T22:28:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test7' 2026-02-22T22:28:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test7 2026-02-22T22:28:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test7 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:08.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test7 2026-02-22T22:28:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test7 2026-02-22T22:28:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test7 '' 2026-02-22T22:28:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:08.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test7.status 2026-02-22T22:28:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test7.status 2026-02-22T22:28:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test7 2026-02-22T22:28:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:08.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test7 2026-02-22T22:28:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test7 '..*' 2026-02-22T22:28:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test7 2026-02-22T22:28:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test7.status 2026-02-22T22:28:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test7.status 2026-02-22T22:28:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test7 2026-02-22T22:28:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:08.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:08.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:08.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.status 2026-02-22T22:28:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:08.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:28:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test7 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:08.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:08.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:09.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test7 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:09.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test7 2026-02-22T22:28:09.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test7: 2026-02-22T22:28:09.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 26707d20-3b5a-4c06-983c-c59b35d21959 2026-02-22T22:28:09.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:09.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:28:09.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:28:09.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:03' 2026-02-22T22:28:09.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:28:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:28:09.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:28:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:28:09.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:28:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:28:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:28:09.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:09.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:03' 2026-02-22T22:28:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test7 up+replaying '2026-02-22 22:28:03' 2026-02-22T22:28:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:28:03' 2026-02-22T22:28:09.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:09.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:09.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING743514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44044407mirror.1trial19019.2.3-689-gf90f4eaefalseOK44314434mirror.2trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}43514354mirror.0trial1902026-02-22 22:27:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}43514354mirror.0trial1902026-02-22 22:27:44test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:44test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":27603.2,"entries_behind_primary":0,"entries_per_second":6.8,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:27:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:27:52test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:28:03' 2026-02-22T22:28:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test7'\'']/last_update' 2026-02-22T22:28:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:03' 2026-02-22T22:28:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test7'\'']/state' 2026-02-22T22:28:09.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:28:09.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:28:09.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:28:09.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:28:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:28:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test7 down+unknown 2026-02-22T22:28:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:09.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:09.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test7 down+unknown '' '' 2026-02-22T22:28:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:10.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:10.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test7 2026-02-22T22:28:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test7: 2026-02-22T22:28:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 26707d20-3b5a-4c06-983c-c59b35d21959 2026-02-22T22:28:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:28:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:28:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:28:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:28:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:28:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:28:10.442 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:03' 2026-02-22T22:28:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:28:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:28:10.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:28:10.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:28:10.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:10.445 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:28:10.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:28:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:10.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test7 down+unknown '' 2026-02-22T22:28:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:10.450 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:28:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:10.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK7test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":41404.8,"entries_behind_primary":0,"entries_per_second":10.2,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:27:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:27:44test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:44test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":27603.2,"entries_behind_primary":0,"entries_per_second":6.8,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:27:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:27:52test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:28:03' 2026-02-22T22:28:10.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test7'\'']/last_update' 2026-02-22T22:28:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test7'\'']/state' 2026-02-22T22:28:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:28:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:28:10.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:28:10.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:10.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:10.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test7 2026-02-22T22:28:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test7 2026-02-22T22:28:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:28:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test7.export 2026-02-22T22:28:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test7.export 2026-02-22T22:28:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.export 2026-02-22T22:28:10.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test7.export 2026-02-22T22:28:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test7.export 2026-02-22T22:28:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test7.export 2026-02-22T22:28:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test7.export 2026-02-22T22:28:10.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test7 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.export 2026-02-22T22:28:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test7 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test7.export 2026-02-22T22:28:10.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:10.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test7.export 2026-02-22T22:28:10.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test7.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test7.export 2026-02-22T22:28:10.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:10.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: release leader and test it is acquired by secondary' 2026-02-22T22:28:10.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:28:10.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:28:10.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:28:10 TEST: release leader and test it is acquired by secondary 2026-02-22T22:28:10.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:28:10 TEST: release leader and test it is acquired by secondary 2026-02-22T22:28:10.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:28:10.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:10.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:10.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:10.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:10.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:28:10.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:10.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:10.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:10.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:10.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:10.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:28:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:28:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ release_leader 2026-02-22T22:28:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror leader release' 2026-02-22T22:28:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror leader release 2026-02-22T22:28:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:10.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:10.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:10.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror leader release 2026-02-22T22:28:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader_released 2026-02-22T22:28:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local i 2026-02-22T22:28:11.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 0 2026-02-22T22:28:11.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 10 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in `seq 10` 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:28:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:28:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:28:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:28:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:28:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:28:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:12.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:12.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:12.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:28:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:28:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:12.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:12.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:28:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:28:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=1 2026-02-22T22:28:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test2 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:1 mirror test1 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:1 mirror test1 started 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:13.277 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:1 mirror test1 started 2026-02-22T22:28:13.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:13.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:13.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:13.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:28:13.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:28:13.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:28:13.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:28:13.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:28:13.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:1 2026-02-22T22:28:13.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:13.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:13.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=1 2026-02-22T22:28:13.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:28:13.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:13.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:28:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:28:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:28:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:13.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:28:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:28:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:28:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:13.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:13.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T22:28:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:28:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:28:13.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:28:13.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:13.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:28:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:14.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T22:28:14.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:28:14.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:28:14.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:28:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 2 2026-02-22T22:28:16.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:28:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:28:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T22:28:16.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:28:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:28:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:28:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:28:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:28:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:28:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:28:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:28:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:28:16.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:28:16.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:16.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:28:16.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:28:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:28:16.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:28:16.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:28:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:1 cluster2 mirror test1 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:1 cluster2 mirror test1 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:28:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:28:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:16.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:28:16.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:16.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:16.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:16.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:16.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:16.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:16.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:28:16.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:28:16.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:16.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:16.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:16.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:16.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:28:16.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:16.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:16.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:16.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:16.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=9, entry_tid=101' -a 'object_number=1, tag_tid=9, entry_tid=101' = 'object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=8, entry_tid=101' '!=' '' 2026-02-22T22:28:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=8, entry_tid=101' = '' 2026-02-22T22:28:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:16.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:16.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:16.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=9 2026-02-22T22:28:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=8 2026-02-22T22:28:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:28:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:16.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:28:16.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 9 = 8 -a 101 -le 101 2026-02-22T22:28:16.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:16.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:16.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:17.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:28:17.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:17.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:17.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:17.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:17.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:17.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:28:17.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:17.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:17.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:17.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:17.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:17.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=9, entry_tid=101' -a 'object_number=1, tag_tid=9, entry_tid=101' = 'object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=8, entry_tid=101' '!=' 'object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:28:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:28:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:28:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:28:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:28:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:17.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=8, entry_tid=101], [object_number=0, tag_tid=8, entry_tid=100], [object_number=3, tag_tid=8, entry_tid=99], [object_number=2, tag_tid=8, entry_tid=98]]], state=connected] 2026-02-22T22:28:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=9, entry_tid=101' -a 'object_number=1, tag_tid=9, entry_tid=101' = 'object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=8, entry_tid=101' '!=' 'object_number=1, tag_tid=8, entry_tid=101' 2026-02-22T22:28:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:28:18.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:28:18.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:18.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:18.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:18.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:18.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:18.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:18.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:28:18.749 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:18.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:18.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:18.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:18.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:18.794 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:18.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:18.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:18.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:18.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:18.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:18.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=9, entry_tid=101' -a 'object_number=1, tag_tid=9, entry_tid=101' = 'object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:28:18.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:28:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:28:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:28:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:28:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:19.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:28:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:28:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:28:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}} 2026-02-22T22:28:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.1 on trial190 2026-02-22T22:28:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:18' 2026-02-22T22:28:19.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:28:19.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:28:19.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:28:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}} 2026-02-22T22:28:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.1 on trial190' ']' 2026-02-22T22:28:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.1 on trial190' 2026-02-22T22:28:19.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.1 on trial190 2026-02-22T22:28:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:19.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:18' 2026-02-22T22:28:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:28:18' 2026-02-22T22:28:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:28:18' 2026-02-22T22:28:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING743514354mirror.0trial19019.2.3-689-gf90f4eaefalseOK44044407mirror.1trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}}44044407mirror.1trial1902026-02-22 22:28:18test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}44044407mirror.1trial1902026-02-22 22:28:18test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44044407mirror.1trial1902026-02-22 22:28:18test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44044407mirror.1trial1902026-02-22 22:28:18test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44044407mirror.1trial1902026-02-22 22:28:18test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44044407mirror.1trial1902026-02-22 22:28:18test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44044407mirror.1trial1902026-02-22 22:28:18' 2026-02-22T22:28:19.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:19.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:28:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:18' 2026-02-22T22:28:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:28:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:28:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:28:19.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:28:19.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:20.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:28:20.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:20.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:20.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:20.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:20.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:20.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:20.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:20.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:20.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}} 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:18' 2026-02-22T22:28:20.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:28:20.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:28:20.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:28:20.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:28:20.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:20.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:20.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:28:20.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:28:20.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:20.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:20.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:20.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:28:20.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:28:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK7test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}}2026-02-22 22:28:18test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:28:18test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:18test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:18test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:18test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:18test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:18' 2026-02-22T22:28:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:21.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:28:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:28:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:28:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:21.033 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:28:21.034 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:28:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:28:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:21.035 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:28:21.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:28:21.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:28:21.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:21.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:28:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:28:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:28:21.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:21.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:28:21.153 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:21.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:21.286 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:21.290 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:28:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:1 mirror test2 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:1 mirror test2 started 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:21.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:1 mirror test2 started 2026-02-22T22:28:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:28:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:28:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:28:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:1 rbd mirror status mirror/test2 2026-02-22T22:28:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:28:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:1 2026-02-22T22:28:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=1 2026-02-22T22:28:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:28:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:22.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:28:22.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:22.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test2 2026-02-22T22:28:22.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:22.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:22.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:28:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:28:22.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:22.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:22.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test2 2026-02-22T22:28:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:28:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:28:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:28:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:28:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:28:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:28:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:28:22.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T22:28:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:28:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:28:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:28:22.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:28:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:28:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:28:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:28:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:1 cluster2 mirror test2 2026-02-22T22:28:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:1 cluster2 mirror test2 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:28:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:28:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:28:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:28:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:28:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:28:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:22.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:28:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:22.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:28:22.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:28:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:28:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:28:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:28:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:22.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:28:22.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:28:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:28:22.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:22.851 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:28:22.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:28:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:28:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:28:22.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:28:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:22.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:28:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:28:22.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:28:22.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:22.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' '!=' '' 2026-02-22T22:28:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' = '' 2026-02-22T22:28:22.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:28:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:22.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=5 2026-02-22T22:28:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=4 2026-02-22T22:28:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:28:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:28:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:28:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 5 = 4 -a 101 -le 101 2026-02-22T22:28:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:28:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:23.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:23.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:23.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:23.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:23.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:23.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:23.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:28:23.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:28:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:28:23.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:23.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:23.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:23.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:28:23.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:28:23.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:28:23.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:28:23.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:23.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:23.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:23.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:28:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:28:23.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:28:23.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:23.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:28:23.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:28:23.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:28:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' '!=' 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:28:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:28:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:23.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:23.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:28:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:23.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:23.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:23.678 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:28:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:28:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:28:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:28:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:28:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:28:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:28:23.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:28:23.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:28:23.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:28:23.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:23.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:23.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:23.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:28:23.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:28:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:28:23.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:28:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:23.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:23.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:23.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:28:23.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T22:28:23.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' '!=' 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T22:28:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:28:24.607 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test2 2026-02-22T22:28:24.607 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:24.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test2 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:24.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:24.610 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:24.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:24.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:24.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:24.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test2 2026-02-22T22:28:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:28:24.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:28:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:24.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:24.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:28:24.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:28:24.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:28:24.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:28:24.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:28:24.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:28:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:28:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:28:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:28:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:28:24.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:28:24.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:28:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:28:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:28:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:28:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test2 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:28:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:24.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:28:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:25.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test2 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:28:25.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:25.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:25.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:25.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:25.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:25.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:28:25.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:25.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:25.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test2 2026-02-22T22:28:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test2: 2026-02-22T22:28:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0 2026-02-22T22:28:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}} 2026-02-22T22:28:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.1 on trial190 2026-02-22T22:28:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:18' 2026-02-22T22:28:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:28:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:28:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:28:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}} 2026-02-22T22:28:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.1 on trial190' ']' 2026-02-22T22:28:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.1 on trial190' 2026-02-22T22:28:25.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.1 on trial190 2026-02-22T22:28:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:25.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:25.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:18' 2026-02-22T22:28:25.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test2 up+replaying '2026-02-22 22:28:18' 2026-02-22T22:28:25.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:25.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:25.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:25.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:25.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:28:18' 2026-02-22T22:28:25.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:25.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:25.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING743514354mirror.0trial19019.2.3-689-gf90f4eaefalseOK44044407mirror.1trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}}44044407mirror.1trial1902026-02-22 22:28:18test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}44044407mirror.1trial1902026-02-22 22:28:18test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44044407mirror.1trial1902026-02-22 22:28:18test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44044407mirror.1trial1902026-02-22 22:28:18test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44044407mirror.1trial1902026-02-22 22:28:18test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44044407mirror.1trial1902026-02-22 22:28:18test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44044407mirror.1trial1902026-02-22 22:28:18' 2026-02-22T22:28:25.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:25.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/last_update' 2026-02-22T22:28:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:18' 2026-02-22T22:28:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/state' 2026-02-22T22:28:25.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:28:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:28:25.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:28:25.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:28:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:28:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test2 down+unknown 2026-02-22T22:28:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:25.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test2 down+unknown '' '' 2026-02-22T22:28:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:26.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:26.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test2 2026-02-22T22:28:26.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test2: 2026-02-22T22:28:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0 2026-02-22T22:28:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:28:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:28:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:28:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:28:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:28:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}} 2026-02-22T22:28:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:18' 2026-02-22T22:28:26.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:28:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:28:26.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:28:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:28:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:26.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:28:26.902 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:28:26.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:26.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test2 down+unknown '' 2026-02-22T22:28:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:28:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK7test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}}2026-02-22 22:28:18test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 22:28:18test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:18test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:18test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:18test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:18test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:18' 2026-02-22T22:28:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:26.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/last_update' 2026-02-22T22:28:26.949 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/state' 2026-02-22T22:28:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:28:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:28:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:28:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test2 2026-02-22T22:28:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:28:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:28:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test2.export 2026-02-22T22:28:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test2.export 2026-02-22T22:28:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export 2026-02-22T22:28:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test2.export 2026-02-22T22:28:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test2.export 2026-02-22T22:28:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:28:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:28:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test2 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export 2026-02-22T22:28:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:27.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test2 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:28:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:28:27.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:28:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: stop leader and test replay' 2026-02-22T22:28:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:28:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:28:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:28:27 TEST: stop leader and test replay 2026-02-22T22:28:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:28:27 TEST: stop leader and test replay 2026-02-22T22:28:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:1 2026-02-22T22:28:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:28:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:1 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:28:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:28:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:28:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:28:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:28:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:28:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:28:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=28024 2026-02-22T22:28:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 28024 ']' 2026-02-22T22:28:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill 28024 2026-02-22T22:28:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:28:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:27.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:28:27.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=28024 '$2 == pid {print; exit 1}' 2026-02-22T22:28:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:ubuntu 28024 0.7 0.0 553836 51828 ? Ssl 22:27 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:28:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:28:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:28:28.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=28024 '$2 == pid {print; exit 1}' 2026-02-22T22:28:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:28:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=28024 '$2 == pid {print; exit 1}' 2026-02-22T22:28:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:28.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:28.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T22:28:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:28.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:28.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:1 2026-02-22T22:28:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:28.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:28.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:28:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:28:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:28:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image8=test8 2026-02-22T22:28:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test8 2026-02-22T22:28:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:28:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:28.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test8 2026-02-22T22:28:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ prev_leader=1 2026-02-22T22:28:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:28:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:28:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:28:28.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:29.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:28:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:28:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:28:29.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:29.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:29.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:29.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:29.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:28:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=0 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test8 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test1 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:28:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:28:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:28:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:28:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:28:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:28:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:28:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:28:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:28:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:28:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:28:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:28:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:28:30.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:28:30.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:28:30.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:28:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:28:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:28:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:28:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:28:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:28:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:28:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:31.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:28:31.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:28:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:28:31.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:28:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:28:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:28:31.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:28:31.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:28:31.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:28:31.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:28:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:28:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:28:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:28:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:28:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:28:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:28:31.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:28:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:28:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:28:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:28:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:31.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:28:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:28:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:31.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:31.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:31.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:31.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:31.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:32.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:32.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:32.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=10, entry_tid=101' -a 'object_number=1, tag_tid=10, entry_tid=101' = 'object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:32.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=9, entry_tid=101' '!=' '' 2026-02-22T22:28:32.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:32.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=9, entry_tid=101' = '' 2026-02-22T22:28:32.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=10 2026-02-22T22:28:32.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:32.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:32.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:32.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=9 2026-02-22T22:28:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:32.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:28:32.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:32.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:32.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:28:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 10 = 9 -a 101 -le 101 2026-02-22T22:28:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:32.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:32.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:32.434 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:32.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:32.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:32.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:32.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:32.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:32.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:32.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:32.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:32.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:32.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:32.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:32.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:32.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:32.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=10, entry_tid=101' -a 'object_number=1, tag_tid=10, entry_tid=101' = 'object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=9, entry_tid=101' '!=' 'object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:28:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:32.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:32.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:32.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:33.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:33.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:33.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:33.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:33.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:33.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=9, entry_tid=101], [object_number=0, tag_tid=9, entry_tid=100], [object_number=3, tag_tid=9, entry_tid=99], [object_number=2, tag_tid=9, entry_tid=98]]], state=connected] 2026-02-22T22:28:33.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=10, entry_tid=101' -a 'object_number=1, tag_tid=10, entry_tid=101' = 'object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:33.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=9, entry_tid=101' '!=' 'object_number=1, tag_tid=9, entry_tid=101' 2026-02-22T22:28:33.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:33.087 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:28:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:33.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:33.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:33.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:33.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:33.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:33.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:33.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:33.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:33.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:34.001 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:34.001 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:34.001 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:34.001 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:34.001 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:34.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:34.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:34.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:34.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:34.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:34.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:34.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:34.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:34.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=10, entry_tid=101' -a 'object_number=1, tag_tid=10, entry_tid=101' = 'object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:34.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:28:34.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:35.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:28:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:28:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:28:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}} 2026-02-22T22:28:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:28:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:31' 2026-02-22T22:28:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:28:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:28:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:28:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}} 2026-02-22T22:28:35.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:28:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:28:35.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:28:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:35.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:35.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:31' 2026-02-22T22:28:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:28:31' 2026-02-22T22:28:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:28:31' 2026-02-22T22:28:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING843514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}}43514354mirror.0trial1902026-02-22 22:28:31test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43514354mirror.0trial1902026-02-22 22:28:31test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:28:31test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:28:31test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:31test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:31test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:31test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:28:32' 2026-02-22T22:28:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:28:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:31' 2026-02-22T22:28:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:28:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:28:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:28:35.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:28:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:28:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:28:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:28:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:35.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:28:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:36.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:36.159 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:28:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:28:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:28:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:28:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:28:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:28:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:28:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:28:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}} 2026-02-22T22:28:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:31' 2026-02-22T22:28:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:28:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:28:36.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:28:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:28:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:28:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:28:36.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:36.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:36.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:36.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:28:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:28:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK8test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}}2026-02-22 22:28:31test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:28:31test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:31test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:31test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:31test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:31test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:31test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:28:32' 2026-02-22T22:28:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:36.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:28:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:28:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:28:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:28:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:28:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:28:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:28:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:28:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:28:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:28:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:28:36.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:28:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:28:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:36.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:36.518 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:28:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test8 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test8 started 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test8 started 2026-02-22T22:28:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:28:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:28:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:28:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test8 2026-02-22T22:28:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:28:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:28:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:28:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:28:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:28:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test8 2026-02-22T22:28:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:28:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:28:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:28:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:37.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:37.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test8 2026-02-22T22:28:37.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:28:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:28:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:28:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:28:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:28:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:28:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:28:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test8 100 2026-02-22T22:28:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:28:37.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:28:37.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:37.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:28:37.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test8 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:28:37.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:28:37.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:28:37.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:28:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test8 2026-02-22T22:28:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:28:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test8 2026-02-22T22:28:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:28:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:28:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:37.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test8 2026-02-22T22:28:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test8 ']' 2026-02-22T22:28:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test8' 2026-02-22T22:28:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test8 2026-02-22T22:28:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test8 2026-02-22T22:28:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test8 2026-02-22T22:28:38.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test8 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test8 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test8 '' 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test8 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test8.status 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test8.status 2026-02-22T22:28:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:38.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test8 2026-02-22T22:28:38.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=0, tag_tid=2, entry_tid=44], [object_number=3, tag_tid=2, entry_tid=43], [object_number=2, tag_tid=2, entry_tid=42], [object_number=1, tag_tid=2, entry_tid=41]]], state=connected] 2026-02-22T22:28:38.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test8 2026-02-22T22:28:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test8 2026-02-22T22:28:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test8 '..*' 2026-02-22T22:28:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test8 2026-02-22T22:28:38.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:38.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test8.status 2026-02-22T22:28:38.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test8.status 2026-02-22T22:28:38.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test8 2026-02-22T22:28:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:38.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:38.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:38.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:38.112 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=0, tag_tid=2, entry_tid=44], [object_number=3, tag_tid=2, entry_tid=43], [object_number=2, tag_tid=2, entry_tid=42], [object_number=1, tag_tid=2, entry_tid=41]]], state=connected] 2026-02-22T22:28:38.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=0, tag_tid=2, entry_tid=44' 2026-02-22T22:28:38.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=0, tag_tid=2, entry_tid=44' 2026-02-22T22:28:38.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=0, tag_tid=2, entry_tid=44' '!=' '' 2026-02-22T22:28:38.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:38.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=0, tag_tid=2, entry_tid=44' = '' 2026-02-22T22:28:38.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=0, tag_tid=2, entry_tid=44' 2026-02-22T22:28:38.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:38.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:38.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:38.118 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:28:38.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=0, tag_tid=2, entry_tid=44' 2026-02-22T22:28:38.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:38.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=2 2026-02-22T22:28:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:28:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=0, tag_tid=2, entry_tid=44' 2026-02-22T22:28:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=44 2026-02-22T22:28:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 2 -a 101 -le 44 2026-02-22T22:28:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test8 2026-02-22T22:28:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test8 ']' 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test8' 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test8 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test8 2026-02-22T22:28:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test8 2026-02-22T22:28:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test8 2026-02-22T22:28:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test8 2026-02-22T22:28:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test8 '' 2026-02-22T22:28:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:38.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test8 2026-02-22T22:28:38.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:38.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test8.status 2026-02-22T22:28:38.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test8.status 2026-02-22T22:28:38.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:38.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test8 2026-02-22T22:28:38.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:38.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:38.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:38.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:38.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:38.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test8 2026-02-22T22:28:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test8 2026-02-22T22:28:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test8 '..*' 2026-02-22T22:28:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test8 2026-02-22T22:28:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test8.status 2026-02-22T22:28:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test8.status 2026-02-22T22:28:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test8 2026-02-22T22:28:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.status 2026-02-22T22:28:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:28:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test8 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test8 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:39.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:39.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test8 2026-02-22T22:28:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test8: 2026-02-22T22:28:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: cf8c8374-a30a-4b73-92b3-fc0829311a96 2026-02-22T22:28:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:28:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:28:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:32' 2026-02-22T22:28:39.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:28:39.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:28:39.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:28:39.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:28:39.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:28:39.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:28:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:28:39.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:39.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:32' 2026-02-22T22:28:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test8 up+replaying '2026-02-22 22:28:32' 2026-02-22T22:28:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:28:32' 2026-02-22T22:28:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:39.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING843514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}}43514354mirror.0trial1902026-02-22 22:28:31test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43514354mirror.0trial1902026-02-22 22:28:31test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:28:31test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:28:31test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:31test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:31test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:31test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:28:32' 2026-02-22T22:28:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:39.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test8'\'']/last_update' 2026-02-22T22:28:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:32' 2026-02-22T22:28:39.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test8'\'']/state' 2026-02-22T22:28:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:28:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:28:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:28:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:28:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:28:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test8 down+unknown 2026-02-22T22:28:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:40.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test8 down+unknown '' '' 2026-02-22T22:28:40.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:40.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test8 2026-02-22T22:28:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test8: 2026-02-22T22:28:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: cf8c8374-a30a-4b73-92b3-fc0829311a96 2026-02-22T22:28:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:28:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:28:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:28:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:28:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:28:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:28:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:32' 2026-02-22T22:28:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:28:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:28:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:28:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:28:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:28:40.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:28:40.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:40.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test8 down+unknown '' 2026-02-22T22:28:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:28:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:40.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK8test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":9}}2026-02-22 22:28:31test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:28:31test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:31test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:31test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:31test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:31test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:31test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:28:32' 2026-02-22T22:28:40.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:40.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test8'\'']/last_update' 2026-02-22T22:28:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test8'\'']/state' 2026-02-22T22:28:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:28:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:28:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:28:40.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:40.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:40.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test8 2026-02-22T22:28:40.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:40.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test8 2026-02-22T22:28:40.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:28:40.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test8.export 2026-02-22T22:28:40.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test8.export 2026-02-22T22:28:40.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:40.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.export 2026-02-22T22:28:40.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test8.export 2026-02-22T22:28:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test8.export 2026-02-22T22:28:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:40.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test8.export 2026-02-22T22:28:40.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test8.export 2026-02-22T22:28:40.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test8 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.export 2026-02-22T22:28:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:41.002 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test8 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test8.export 2026-02-22T22:28:41.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test8.export 2026-02-22T22:28:41.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test8.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test8.export 2026-02-22T22:28:41.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:41.232 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: start previous leader and test replay' 2026-02-22T22:28:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:28:41.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:28:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:28:41 TEST: start previous leader and test replay 2026-02-22T22:28:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:28:41 TEST: start previous leader and test replay 2026-02-22T22:28:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:1 2026-02-22T22:28:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:28:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image9=test9 2026-02-22T22:28:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test9 2026-02-22T22:28:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:28:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test9 2026-02-22T22:28:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test9 2026-02-22T22:28:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:28:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:28:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test1 2026-02-22T22:28:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:28:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:28:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:28:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:41.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:28:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:28:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:28:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:28:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:28:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:28:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:28:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:28:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:28:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:28:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:28:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:28:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:28:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:28:42.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:28:42.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:28:42.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:28:42.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:28:42.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:28:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:28:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:28:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:28:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 5000.09 bytes/sec: 20 MiB/s 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:42.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:42.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:42.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:42.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:42.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:28:42.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:42.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:42.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:42.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:28:42.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:42.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:42.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:42.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=11, entry_tid=101' -a 'object_number=1, tag_tid=11, entry_tid=101' = 'object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:42.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=10, entry_tid=101' '!=' '' 2026-02-22T22:28:42.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:42.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=10, entry_tid=101' = '' 2026-02-22T22:28:42.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:42.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:28:42.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:42.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:42.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=11 2026-02-22T22:28:42.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:42.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:42.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:42.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=10 2026-02-22T22:28:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:28:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:28:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:28:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 11 = 10 -a 101 -le 101 2026-02-22T22:28:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:43.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:43.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:43.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:43.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:43.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:28:43.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:28:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:43.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:28:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:43.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=11, entry_tid=101' -a 'object_number=1, tag_tid=11, entry_tid=101' = 'object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=10, entry_tid=101' '!=' 'object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:43.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:43.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:43.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:43.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:43.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:43.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:43.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:43.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:43.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:43.653 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:28:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:28:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:43.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:43.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:43.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:43.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:28:43.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=10, entry_tid=101], [object_number=0, tag_tid=10, entry_tid=100], [object_number=3, tag_tid=10, entry_tid=99], [object_number=2, tag_tid=10, entry_tid=98]]], state=connected] 2026-02-22T22:28:43.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=11, entry_tid=101' -a 'object_number=1, tag_tid=11, entry_tid=101' = 'object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=10, entry_tid=101' '!=' 'object_number=1, tag_tid=10, entry_tid=101' 2026-02-22T22:28:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:28:44.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:28:44.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:44.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:44.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:44.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:44.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:28:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:28:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:44.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:44.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:44.523 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:44.524 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:44.525 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:44.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:44.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:44.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:44.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:28:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:28:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:28:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:44.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:44.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:44.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:44.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:28:44.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:28:44.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:44.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:28:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:28:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:44.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:28:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:28:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:28:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:28:44.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:44.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:44.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:28:44.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:28:44.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:28:44.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:28:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:28:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=11, entry_tid=101' -a 'object_number=1, tag_tid=11, entry_tid=101' = 'object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:28:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:44.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:28:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:44.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:44.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:28:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:28:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:28:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":34504.0,"entries_behind_primary":101,"entries_per_second":8.5,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":10},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"seconds_until_synced":11} 2026-02-22T22:28:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:28:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:44' 2026-02-22T22:28:45.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:28:45.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:28:45.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:28:45.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":34504.0,"entries_behind_primary":101,"entries_per_second":8.5,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":10},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"seconds_until_synced":11} 2026-02-22T22:28:45.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:28:45.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:28:45.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:28:45.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:45.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:45.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:44' 2026-02-22T22:28:45.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:28:44' 2026-02-22T22:28:45.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:45.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:45.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:45.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:45.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:28:44' 2026-02-22T22:28:45.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:45.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:45.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:45.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGWARNINGWARNING943514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaefalseOK4490mirror.1trial19019.2.3-689-gf90f4eaefalseWARNINGnot reporting statustest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":34504.0,"entries_behind_primary":101,"entries_per_second":8.5,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":10},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"seconds_until_synced":11}43514354mirror.0trial1902026-02-22 22:28:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43514354mirror.0trial1902026-02-22 22:28:44test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:28:44test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:28:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:44test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:44test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":41439.8,"entries_behind_primary":0,"entries_per_second":10.6,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:44test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:28:44' 2026-02-22T22:28:45.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:45.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:28:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:44' 2026-02-22T22:28:45.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:28:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:28:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:28:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:28:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:28:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:28:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:28:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:45.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:45.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:45.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:45.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:28:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:28:46.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:28:46.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:28:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:28:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:28:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:28:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:28:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:28:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":34504.0,"entries_behind_primary":101,"entries_per_second":8.5,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":10},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"seconds_until_synced":11} 2026-02-22T22:28:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:44' 2026-02-22T22:28:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:28:46.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:28:46.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:28:46.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:28:46.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:46.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:46.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:28:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:28:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:28:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:46.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:28:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:46.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK9test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":34504.0,"entries_behind_primary":101,"entries_per_second":8.5,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":10},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"seconds_until_synced":11}2026-02-22 22:28:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:28:44test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:44test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:44test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:44test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":41439.8,"entries_behind_primary":0,"entries_per_second":10.6,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:44test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:28:44' 2026-02-22T22:28:46.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:46.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:28:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:28:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:28:46.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:28:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:28:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:28:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:28:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:28:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:28:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:28:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:28:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:28:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:28:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:46.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:28:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:47.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:47.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:28:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test9 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test9 started 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:28:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:28:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:47.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test9 started 2026-02-22T22:28:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:28:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:28:48.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:28:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test9 2026-02-22T22:28:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:28:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:28:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:28:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:28:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:28:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test9 2026-02-22T22:28:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:28:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:28:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:28:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:28:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:28:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:28:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:28:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:28:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test9 2026-02-22T22:28:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:28:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:28:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:28:48.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:28:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test9 100 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:28:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test9 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:28:48.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:28:48.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:28:48.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:28:48.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test9 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test9 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:48.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:28:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:28:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:48.414 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test9 2026-02-22T22:28:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test9 ']' 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test9' 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test9 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:48.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test9 2026-02-22T22:28:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:48.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:48.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:48.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:48.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test9 2026-02-22T22:28:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test9 2026-02-22T22:28:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test9 '' 2026-02-22T22:28:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test9.status 2026-02-22T22:28:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test9.status 2026-02-22T22:28:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:48.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test9 2026-02-22T22:28:48.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:28:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test9 2026-02-22T22:28:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test9 '..*' 2026-02-22T22:28:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test9.status 2026-02-22T22:28:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test9.status 2026-02-22T22:28:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:48.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test9 2026-02-22T22:28:48.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:28:48.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:28:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:28:48.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:48.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:28:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:48.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=1 2026-02-22T22:28:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:48.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:48.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:28:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=3 2026-02-22T22:28:48.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:28:48.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:48.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:48.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:48.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test9 2026-02-22T22:28:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test9 ']' 2026-02-22T22:28:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test9' 2026-02-22T22:28:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test9 2026-02-22T22:28:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:48.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test9 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:48.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:48.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:49.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:49.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test9 2026-02-22T22:28:49.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test9 2026-02-22T22:28:49.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test9 '' 2026-02-22T22:28:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test9.status 2026-02-22T22:28:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test9.status 2026-02-22T22:28:49.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test9 2026-02-22T22:28:49.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=2, tag_tid=2, entry_tid=46], [object_number=1, tag_tid=2, entry_tid=45], [object_number=0, tag_tid=2, entry_tid=44], [object_number=3, tag_tid=2, entry_tid=43]]], state=connected] 2026-02-22T22:28:49.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test9 2026-02-22T22:28:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test9 '..*' 2026-02-22T22:28:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test9.status 2026-02-22T22:28:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test9.status 2026-02-22T22:28:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test9 2026-02-22T22:28:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:49.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:49.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:49.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:49.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=2, tag_tid=2, entry_tid=46], [object_number=1, tag_tid=2, entry_tid=45], [object_number=0, tag_tid=2, entry_tid=44], [object_number=3, tag_tid=2, entry_tid=43]]], state=connected] 2026-02-22T22:28:49.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=2, tag_tid=2, entry_tid=46' 2026-02-22T22:28:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=2, tag_tid=2, entry_tid=46' 2026-02-22T22:28:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=2, tag_tid=2, entry_tid=46' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=2, tag_tid=2, entry_tid=46' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:28:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=2, tag_tid=2, entry_tid=46' 2026-02-22T22:28:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:28:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=2, tag_tid=2, entry_tid=46' 2026-02-22T22:28:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:28:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=2 2026-02-22T22:28:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:49.142 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=2, tag_tid=2, entry_tid=46' 2026-02-22T22:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:28:49.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=46 2026-02-22T22:28:49.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 2 -a 101 -le 46 2026-02-22T22:28:49.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:28:49.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:28:49.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:28:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test9 2026-02-22T22:28:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test9 ']' 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test9' 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test9 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:28:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test9 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:49.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:49.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test9 2026-02-22T22:28:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test9 2026-02-22T22:28:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test9 '' 2026-02-22T22:28:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:49.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:28:49.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test9.status 2026-02-22T22:28:49.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test9.status 2026-02-22T22:28:49.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:49.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test9 2026-02-22T22:28:49.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:49.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test9 2026-02-22T22:28:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test9 '..*' 2026-02-22T22:28:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:28:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test9 2026-02-22T22:28:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:28:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test9.status 2026-02-22T22:28:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test9.status 2026-02-22T22:28:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test9 2026-02-22T22:28:49.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:28:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:28:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:28:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:49.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:28:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.status 2026-02-22T22:28:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:28:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:28:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test9 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test9 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:28:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:50.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:50.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:50.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:50.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:28:50.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:28:50.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:50.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:50.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test9 2026-02-22T22:28:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test9: 2026-02-22T22:28:50.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 4728bd8a-2d02-4dec-90a0-478e08066247 2026-02-22T22:28:50.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:50.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:28:50.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:28:50.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:44' 2026-02-22T22:28:50.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:28:50.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:28:50.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:28:50.547 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:28:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:28:50.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:28:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:28:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:50.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:44' 2026-02-22T22:28:50.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test9 up+replaying '2026-02-22 22:28:44' 2026-02-22T22:28:50.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:28:50.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:50.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:50.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:28:50.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:28:44' 2026-02-22T22:28:50.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:50.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:50.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING943514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaefalseOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":34504.0,"entries_behind_primary":101,"entries_per_second":8.5,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":10},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"seconds_until_synced":11}43514354mirror.0trial1902026-02-22 22:28:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43514354mirror.0trial1902026-02-22 22:28:44test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:28:44test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}43514354mirror.0trial1902026-02-22 22:28:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:44test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:44test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":41439.8,"entries_behind_primary":0,"entries_per_second":10.6,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}43514354mirror.0trial1902026-02-22 22:28:44test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43514354mirror.0trial1902026-02-22 22:28:44' 2026-02-22T22:28:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:50.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test9'\'']/last_update' 2026-02-22T22:28:50.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:28:44' 2026-02-22T22:28:50.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test9'\'']/state' 2026-02-22T22:28:50.612 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:28:50.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:28:50.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:28:50.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:28:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:50.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:28:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test9 down+unknown 2026-02-22T22:28:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:28:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:51.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test9 down+unknown '' '' 2026-02-22T22:28:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:28:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:28:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test9 2026-02-22T22:28:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test9: 2026-02-22T22:28:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 4728bd8a-2d02-4dec-90a0-478e08066247 2026-02-22T22:28:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:28:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:28:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:28:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:28:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:28:51.647 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:28:51.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:28:51.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:28:44' 2026-02-22T22:28:51.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:28:51.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:28:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:28:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:28:51.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:28:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:28:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:28:51.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:28:51.654 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test9 down+unknown '' 2026-02-22T22:28:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:28:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:28:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:28:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:28:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:28:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK9test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":34504.0,"entries_behind_primary":101,"entries_per_second":8.5,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":10},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"seconds_until_synced":11}2026-02-22 22:28:44test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:28:44test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:44test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:28:44test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:44test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:44test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:44test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":41439.8,"entries_behind_primary":0,"entries_per_second":10.6,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:28:44test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:28:44' 2026-02-22T22:28:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:28:51.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test9'\'']/last_update' 2026-02-22T22:28:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:28:51.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test9'\'']/state' 2026-02-22T22:28:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:28:51.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:28:51.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:28:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:28:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test9 2026-02-22T22:28:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:51.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test9 2026-02-22T22:28:51.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:28:51.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test9.export 2026-02-22T22:28:51.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test9.export 2026-02-22T22:28:51.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:51.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.export 2026-02-22T22:28:51.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test9.export 2026-02-22T22:28:51.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test9.export 2026-02-22T22:28:51.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:28:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test9.export 2026-02-22T22:28:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test9.export 2026-02-22T22:28:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test9 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.export 2026-02-22T22:28:51.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test9 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test9.export 2026-02-22T22:28:51.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:28:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test9.export 2026-02-22T22:28:52.020 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test9.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test9.export 2026-02-22T22:28:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:28:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: crash leader and test replay' 2026-02-22T22:28:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:28:52.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:28:52.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:28:52 TEST: crash leader and test replay 2026-02-22T22:28:52.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:28:52 TEST: crash leader and test replay 2026-02-22T22:28:52.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:0 -KILL 2026-02-22T22:28:52.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:52.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig=-KILL 2026-02-22T22:28:52.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:52.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:28:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:28:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:0 2026-02-22T22:28:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:28:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:28:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:28:52.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:28:52.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:28:52.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=25231 2026-02-22T22:28:52.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 25231 ']' 2026-02-22T22:28:52.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill -KILL 25231 2026-02-22T22:28:52.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:28:52.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:28:52.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:28:52.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=25231 '$2 == pid {print; exit 1}' 2026-02-22T22:28:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:28:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:28:52.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=25231 '$2 == pid {print; exit 1}' 2026-02-22T22:28:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:28:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:28:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:28:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:28:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:52.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:28:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:28:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:28:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:52.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:28:52.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:28:52.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:28:52.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image10=test10 2026-02-22T22:28:52.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test10 2026-02-22T22:28:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:28:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:28:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:28:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:28:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:28:52.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test10 2026-02-22T22:28:57.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ prev_leader=0 2026-02-22T22:28:57.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:28:57.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:28:57.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:28:57.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:28:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:28:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:58.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:58.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:28:58.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:28:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:58.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:58.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:58.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:58.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:58.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:28:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:28:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:28:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:28:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:28:58.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:28:58.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:28:58.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:28:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:28:58.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:28:58.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:28:58.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:28:58.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:28:58.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:28:58.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:58.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:58.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:28:58.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:28:58.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:58.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:58.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:58.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:28:58.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:58.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:28:58.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:28:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:28:58.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:28:58.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:28:58.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:28:58.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:28:58.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:28:58.481 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:28:58.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:58.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:58.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:28:58.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:28:58.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:58.487 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:28:58.488 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:28:58.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:28:58.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:28:58.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:28:58.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:28:58.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:28:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:28:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:28:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:58.553 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:58.554 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:28:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:28:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:28:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:28:58.556 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:28:58.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:28:58.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:28:58.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:28:58.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:28:58.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:28:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:28:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:28:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:28:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:28:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:28:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:28:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:28:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:28:58.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:28:58.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:28:58.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:28:58.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:28:58.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:28:58.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:58.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:28:58.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:28:59.678 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:28:59.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:59.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:28:59.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:28:59.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:59.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:59.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:59.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:59.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:59.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:28:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:28:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:28:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:28:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:28:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:28:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:28:59.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:28:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:28:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:28:59.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:59.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:28:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:28:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:59.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:59.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:28:59.753 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:28:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:28:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:28:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:28:59.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:59.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:59.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:28:59.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:28:59.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:59.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:59.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:28:59.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:28:59.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:28:59.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:28:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:28:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:28:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:28:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:28:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:28:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:28:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:28:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:28:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:28:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:28:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:28:59.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:28:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:28:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:28:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:28:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:28:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:28:59.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:28:59.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:28:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:28:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:28:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:28:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:28:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:28:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:28:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:28:59.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:28:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:28:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:28:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:28:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:29:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:00.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:00.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:00.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:29:00.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:29:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:29:00.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:29:00.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:29:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:29:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:29:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:29:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:00.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:29:00.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:00.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:29:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:29:00.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:29:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:29:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:29:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 2 2026-02-22T22:29:02.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:29:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:02.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:29:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:29:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:02.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:29:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:29:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:29:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:29:02.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:02.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:02.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:02.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:29:02.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:29:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:29:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:29:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:29:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:29:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:29:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:29:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:29:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:02.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:29:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:02.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:02.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:02.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:02.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:29:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:29:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:29:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:29:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:29:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:29:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:29:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:29:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:29:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:29:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:29:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:29:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:29:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:29:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:29:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:29:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:29:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:29:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:02.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:02.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:29:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:29:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:29:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:29:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:02.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:02.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:02.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:02.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:02.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:29:02.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:29:02.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:29:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:29:02.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:29:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:29:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:29:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:29:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:02.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:02.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:02.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:29:02.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:29:02.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:29:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:29:02.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:02.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:02.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:29:02.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 4 2026-02-22T22:29:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:29:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:29:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:29:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:06.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:29:06.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:29:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:29:06.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:29:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:29:06.692 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:29:06.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:29:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:29:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:29:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:29:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:29:06.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:29:06.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:06.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:29:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:29:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:29:06.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:06.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:06.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:06.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:29:06.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:29:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:29:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:29:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:29:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:29:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:29:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:29:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:29:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:29:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:29:06.972 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:29:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:29:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:06.974 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:06.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:29:06.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:29:06.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:06.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:06.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:29:06.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:06.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:29:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:29:06.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:29:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:29:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:29:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:29:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:07.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:29:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:29:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:29:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:29:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:29:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:29:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:07.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:29:07.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:07.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:29:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 4 2026-02-22T22:29:11.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:29:11.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:11.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:29:11.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:11.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:29:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:29:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:29:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:29:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:29:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:29:11.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:29:11.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:11.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:11.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:29:11.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:29:11.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:29:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:11.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:11.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:11.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:29:11.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:11.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:11.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:11.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:11.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:11.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:11.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:29:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:29:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:29:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:29:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:29:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:29:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:29:11.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:11.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:11.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:11.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:29:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:29:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:29:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:29:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:29:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:29:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:11.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:11.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:29:11.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:29:11.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:29:11.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:11.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:11.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:11.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:29:11.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:11.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:11.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:11.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:29:11.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:29:11.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:11.437 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:29:11.438 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:29:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:29:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:29:11.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:29:11.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:11.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:11.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:11.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:29:11.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:11.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:11.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:11.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:29:11.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:29:11.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:11.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:29:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:29:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:29:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:29:11.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:11.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:11.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:11.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:29:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:29:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 4 2026-02-22T22:29:15.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:29:15.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:15.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:29:15.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:29:15.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:15.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:15.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:15.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:15.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:29:15.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:29:15.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:15.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:15.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:15.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:29:15.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:29:15.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:29:15.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:15.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:15.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:15.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:29:15.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:29:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:29:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:29:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:29:15.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:29:15.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:29:15.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:15.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:15.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:29:15.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:29:15.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:29:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:15.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:15.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:29:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:29:15.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:15.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:29:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:29:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:29:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:29:15.792 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:29:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:29:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:29:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:29:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:29:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:29:15.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:29:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:29:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:29:15.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:15.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:15.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:29:15.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:29:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:29:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:29:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:29:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:29:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:29:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:29:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:29:15.915 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:29:15.959 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:29:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:29:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:29:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:29:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:29:15.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:29:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:29:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:29:16.019 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:29:16.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 4 2026-02-22T22:29:20.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:29:20.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:20.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:29:20.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:29:20.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:20.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:20.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:20.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:29:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:29:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:29:20.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:20.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:29:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:29:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:29:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:29:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:29:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:29:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:20.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:29:20.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:29:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:29:20.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:29:20.100 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:20.100 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:20.100 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:29:20.100 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:29:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:29:20.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:20.167 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:20.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:20.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:20.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:20.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:29:20.233 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:29:20.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=2 2026-02-22T22:29:20.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test10 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:2 mirror test1 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:2 mirror test1 started 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:20.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:20.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:29:20.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:29:20.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:20.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:2 mirror test1 started 2026-02-22T22:29:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:29:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:29:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:29:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:29:21.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:29:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:2 2026-02-22T22:29:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=2 2026-02-22T22:29:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:29:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:29:21.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:29:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:29:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:29:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:21.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:29:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:29:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:29:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:29:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:29:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:21.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T22:29:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:29:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:29:21.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:29:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:29:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:29:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:29:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:29:22.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:29:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:22.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T22:29:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:29:22.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Starting"' 2026-02-22T22:29:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Starting"' 2026-02-22T22:29:22.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:29:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = stopped 2026-02-22T22:29:22.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:22.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 2 2026-02-22T22:29:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:2 mirror test1 started 2026-02-22T22:29:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:29:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:29:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:29:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:29:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:29:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:2 2026-02-22T22:29:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=2 2026-02-22T22:29:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:29:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:29:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:29:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:29:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:29:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:29:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:29:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:29:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:29:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:24.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:24.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:24.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T22:29:24.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:29:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:29:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:29:24.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:29:24.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:29:24.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:29:24.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:29:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:29:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:29:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:29:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:29:24.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:29:24.512 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:29:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:29:24.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:29:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:2 cluster2 mirror test1 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:2 cluster2 mirror test1 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:29:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:29:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:24.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:24.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:24.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:29:24.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:24.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:24.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:24.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:24.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:24.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:29:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=12, entry_tid=101' -a 'object_number=1, tag_tid=12, entry_tid=101' = 'object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=11, entry_tid=101' '!=' '' 2026-02-22T22:29:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:29:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=11, entry_tid=101' = '' 2026-02-22T22:29:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=12 2026-02-22T22:29:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=11 2026-02-22T22:29:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:29:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:29:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 12 = 11 -a 101 -le 101 2026-02-22T22:29:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:25.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:29:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:25.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:25.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:25.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:25.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:25.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:25.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:25.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:25.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:25.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:29:25.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=12, entry_tid=101' -a 'object_number=1, tag_tid=12, entry_tid=101' = 'object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=11, entry_tid=101' '!=' 'object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:25.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:29:25.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:25.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:25.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:25.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:25.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:25.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:25.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:25.652 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:25.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:25.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:25.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:25.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:25.737 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:25.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:25.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:25.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:25.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:29:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:25.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:25.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:25.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:25.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=11, entry_tid=101], [object_number=0, tag_tid=11, entry_tid=100], [object_number=3, tag_tid=11, entry_tid=99], [object_number=2, tag_tid=11, entry_tid=98]]], state=connected] 2026-02-22T22:29:25.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:25.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:25.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=12, entry_tid=101' -a 'object_number=1, tag_tid=12, entry_tid=101' = 'object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:25.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=11, entry_tid=101' '!=' 'object_number=1, tag_tid=11, entry_tid=101' 2026-02-22T22:29:25.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:25.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:29:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:26.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:26.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:26.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:26.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:26.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:26.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:26.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:26.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:26.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:26.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:26.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:26.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:26.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:26.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:26.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:26.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=12, entry_tid=101' -a 'object_number=1, tag_tid=12, entry_tid=101' = 'object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:26.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:26.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:29:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:26.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:27.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:27.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:29:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:29:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:29:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11}} 2026-02-22T22:29:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.2 on trial190 2026-02-22T22:29:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:23' 2026-02-22T22:29:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:29:27.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:29:27.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:29:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11}} 2026-02-22T22:29:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.2 on trial190' ']' 2026-02-22T22:29:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.2 on trial190' 2026-02-22T22:29:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.2 on trial190 2026-02-22T22:29:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:27.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:23' 2026-02-22T22:29:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:29:23' 2026-02-22T22:29:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:29:23' 2026-02-22T22:29:27.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:27.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:27.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1043514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaetrueOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11}}44314434mirror.2trial1902026-02-22 22:29:23test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:21test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}44314434mirror.2trial1902026-02-22 22:29:23test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:23test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:23test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:23test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:23test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:23test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:23test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:23' 2026-02-22T22:29:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:29:27.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:23' 2026-02-22T22:29:27.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:29:27.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:29:27.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:29:27.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:29:27.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:29:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:29:27.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:29:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:29:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:29:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:29:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:29:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:29:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:29:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:29:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:29:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:29:28.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11}} 2026-02-22T22:29:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:23' 2026-02-22T22:29:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:29:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:29:28.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:29:28.920 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:29:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:29:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:28.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:29:28.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:29:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:28.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:29:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:29:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:28.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK10test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":11}}2026-02-22 22:29:23test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:21test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:29:23test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:23test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:23test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:23test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:23test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:23test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:23test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:23' 2026-02-22T22:29:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:29:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:28.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:29:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:29:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:29:28.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:29:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:29:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:28.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:28.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:29:28.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:29:28.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:29:28.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:28.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:29:28.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:29:28.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:29:28.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:29:29.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:29.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:29.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:2 mirror test10 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:2 mirror test10 started 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:29.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:2 mirror test10 started 2026-02-22T22:29:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:29:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:29:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:29:30.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:2 rbd mirror status mirror/test10 2026-02-22T22:29:30.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:29:30.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:2 2026-02-22T22:29:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=2 2026-02-22T22:29:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:29:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:29:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test10 2026-02-22T22:29:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:29:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:29:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:29:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:29:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test10 2026-02-22T22:29:30.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:29:30.452 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:29:30.452 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:29:30.452 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:29:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:29:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:29:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:29:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test10 100 2026-02-22T22:29:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:29:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:29:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:29:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test10 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:29:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:29:30.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:29:30.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 5000.09 bytes/sec: 20 MiB/s 2026-02-22T22:29:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:2 cluster2 mirror test10 2026-02-22T22:29:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:29:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:2 cluster2 mirror test10 2026-02-22T22:29:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:29:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:29:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test10 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test10 ']' 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test10' 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test10 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:30.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test10 2026-02-22T22:29:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:30.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test10 2026-02-22T22:29:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test10 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test10 '' 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test10.status 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test10.status 2026-02-22T22:29:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:30.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:30.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test10 2026-02-22T22:29:30.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:30.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:30.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:29:30.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test10 2026-02-22T22:29:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test10 '..*' 2026-02-22T22:29:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test10.status 2026-02-22T22:29:30.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test10.status 2026-02-22T22:29:30.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test10 2026-02-22T22:29:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:30.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:29:30.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:29:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:29:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:29:30.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:30.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:30.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:29:30.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:30.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:30.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:30.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=1 2026-02-22T22:29:30.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:30.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:30.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:30.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:29:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:30.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=3 2026-02-22T22:29:30.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:29:30.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:30.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:30.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test10 2026-02-22T22:29:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test10 ']' 2026-02-22T22:29:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test10' 2026-02-22T22:29:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test10 2026-02-22T22:29:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:31.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test10 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:31.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:31.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:31.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:31.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:31.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:31.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:31.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:31.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:31.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:31.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:31.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:31.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test10 2026-02-22T22:29:31.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:31.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test10 2026-02-22T22:29:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test10 '' 2026-02-22T22:29:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:31.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test10.status 2026-02-22T22:29:31.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test10.status 2026-02-22T22:29:31.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:31.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test10 2026-02-22T22:29:31.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:31.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=95], [object_number=2, tag_tid=2, entry_tid=94], [object_number=1, tag_tid=2, entry_tid=93], [object_number=0, tag_tid=2, entry_tid=92]]], state=connected] 2026-02-22T22:29:31.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test10 2026-02-22T22:29:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test10 '..*' 2026-02-22T22:29:31.208 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:31.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:31.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:31.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:31.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test10.status 2026-02-22T22:29:31.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test10.status 2026-02-22T22:29:31.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test10 2026-02-22T22:29:31.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:31.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=95], [object_number=2, tag_tid=2, entry_tid=94], [object_number=1, tag_tid=2, entry_tid=93], [object_number=0, tag_tid=2, entry_tid=92]]], state=connected] 2026-02-22T22:29:31.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=95' 2026-02-22T22:29:31.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=2, entry_tid=95' 2026-02-22T22:29:31.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=95' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:31.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:29:31.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=95' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:31.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=3, tag_tid=2, entry_tid=95' 2026-02-22T22:29:31.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:31.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:31.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:31.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:29:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=95' 2026-02-22T22:29:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:31.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=2 2026-02-22T22:29:31.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:31.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:31.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:29:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=95' 2026-02-22T22:29:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=95 2026-02-22T22:29:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 2 -a 101 -le 95 2026-02-22T22:29:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:31.454 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test10 2026-02-22T22:29:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test10 ']' 2026-02-22T22:29:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test10' 2026-02-22T22:29:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test10 2026-02-22T22:29:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test10 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:31.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:31.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:31.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:31.465 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test10 2026-02-22T22:29:31.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test10 2026-02-22T22:29:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:31.536 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test10 '' 2026-02-22T22:29:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test10.status 2026-02-22T22:29:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test10.status 2026-02-22T22:29:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test10 2026-02-22T22:29:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:31.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:31.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:31.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:31.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:31.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:31.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test10 2026-02-22T22:29:31.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test10 '..*' 2026-02-22T22:29:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test10 2026-02-22T22:29:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test10.status 2026-02-22T22:29:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test10.status 2026-02-22T22:29:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:31.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test10 2026-02-22T22:29:31.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:31.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.status 2026-02-22T22:29:31.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:31.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:31.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:29:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test10 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test10 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test10 2026-02-22T22:29:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test10: 2026-02-22T22:29:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 8c346bd9-9262-4edd-8b07-709d0b842405 2026-02-22T22:29:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:29:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.2 on trial190 2026-02-22T22:29:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:29' 2026-02-22T22:29:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:29:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:29:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:29:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:29:32.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.2 on trial190' ']' 2026-02-22T22:29:32.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.2 on trial190' 2026-02-22T22:29:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.2 on trial190 2026-02-22T22:29:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:29' 2026-02-22T22:29:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test10 up+replaying '2026-02-22 22:29:29' 2026-02-22T22:29:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:29:29' 2026-02-22T22:29:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:32.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:32.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1043514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaetrueOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}}44314434mirror.2trial1902026-02-22 22:29:29test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:29test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}44314434mirror.2trial1902026-02-22 22:29:29test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:29test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:29test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29' 2026-02-22T22:29:32.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:32.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test10'\'']/last_update' 2026-02-22T22:29:32.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:29' 2026-02-22T22:29:32.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test10'\'']/state' 2026-02-22T22:29:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:29:32.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:29:32.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:29:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:29:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:29:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test10 down+unknown 2026-02-22T22:29:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test10 down+unknown '' '' 2026-02-22T22:29:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test10 2026-02-22T22:29:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test10: 2026-02-22T22:29:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 8c346bd9-9262-4edd-8b07-709d0b842405 2026-02-22T22:29:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:29:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:29:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:29:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:29:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:29:33.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:33.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:29:33.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:29' 2026-02-22T22:29:33.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:29:33.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:29:33.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:29:33.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:29:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:29:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:33.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:29:33.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:29:33.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:33.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test10 down+unknown '' 2026-02-22T22:29:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:29:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK10test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}}2026-02-22 22:29:29test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:29test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:29:29test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:29test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:29test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29' 2026-02-22T22:29:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test10'\'']/last_update' 2026-02-22T22:29:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test10'\'']/state' 2026-02-22T22:29:33.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:29:33.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:33.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:29:33.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:29:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:33.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test10 2026-02-22T22:29:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test10 2026-02-22T22:29:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:29:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test10.export 2026-02-22T22:29:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test10.export 2026-02-22T22:29:33.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.export 2026-02-22T22:29:33.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test10.export 2026-02-22T22:29:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test10.export 2026-02-22T22:29:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:33.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test10.export 2026-02-22T22:29:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test10.export 2026-02-22T22:29:33.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test10 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.export 2026-02-22T22:29:33.944 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:33.949 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test10 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test10.export 2026-02-22T22:29:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test10.export 2026-02-22T22:29:34.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test10.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test10.export 2026-02-22T22:29:34.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:34.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: start previous leader and test replay' 2026-02-22T22:29:34.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:29:34.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:29:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:29:34 TEST: start previous leader and test replay 2026-02-22T22:29:34.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:29:34 TEST: start previous leader and test replay 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:0 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:34.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:34.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:34.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:29:34.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:34.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:34.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:29:34.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image11=test11 2026-02-22T22:29:34.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test11 2026-02-22T22:29:34.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:34.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:29:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:29:34.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test11 2026-02-22T22:29:34.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test11 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:2 mirror test1 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:2 mirror test1 started 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:29:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:29:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:35.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:2 mirror test1 started 2026-02-22T22:29:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:29:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:29:35.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:29:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:29:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:2 2026-02-22T22:29:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:29:35.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=2 2026-02-22T22:29:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:29:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:35.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:29:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:29:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:29:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:35.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:29:35.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:29:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:29:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:29:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:29:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:29:35.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:29:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:29:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:29:35.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:35.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:35.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T22:29:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:29:35.338 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:29:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:29:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:29:35.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:29:35.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:29:35.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:29:35.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:35.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:29:35.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:35.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:29:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:29:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:29:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:29:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:29:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:29:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:29:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:2 cluster2 mirror test1 2026-02-22T22:29:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:29:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:2 cluster2 mirror test1 2026-02-22T22:29:35.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:29:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:29:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:35.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:35.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:35.692 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:35.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:35.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:35.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:35.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:35.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:35.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:35.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:35.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:35.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:35.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:35.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:35.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:35.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:35.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:35.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:35.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:35.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:35.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:35.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:35.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:35.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=13, entry_tid=101' -a 'object_number=1, tag_tid=13, entry_tid=101' = 'object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:35.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=12, entry_tid=101' '!=' '' 2026-02-22T22:29:35.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:29:35.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=12, entry_tid=101' = '' 2026-02-22T22:29:35.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:35.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:35.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:35.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:35.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=13 2026-02-22T22:29:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:35.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:35.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=12 2026-02-22T22:29:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:35.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:29:35.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:35.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:35.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:35.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:29:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 13 = 12 -a 101 -le 101 2026-02-22T22:29:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:35.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:35.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:35.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:36.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:36.127 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:36.127 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:36.127 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=13, entry_tid=101' -a 'object_number=1, tag_tid=13, entry_tid=101' = 'object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=12, entry_tid=101' '!=' 'object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:36.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:36.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:36.605 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.605 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:36.605 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:36.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:36.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:36.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:36.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:36.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:36.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:36.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=12, entry_tid=101], [object_number=0, tag_tid=12, entry_tid=100], [object_number=3, tag_tid=12, entry_tid=99], [object_number=2, tag_tid=12, entry_tid=98]]], state=connected] 2026-02-22T22:29:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=13, entry_tid=101' -a 'object_number=1, tag_tid=13, entry_tid=101' = 'object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=12, entry_tid=101' '!=' 'object_number=1, tag_tid=12, entry_tid=101' 2026-02-22T22:29:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:37.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:37.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:37.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:37.599 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:37.600 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:37.602 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:37.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:37.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:37.603 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:37.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:37.604 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:37.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:37.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:37.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:37.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:37.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:37.634 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=13, entry_tid=101' -a 'object_number=1, tag_tid=13, entry_tid=101' = 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:37.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:37.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:29:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:29:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:29:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:29:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}} 2026-02-22T22:29:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.2 on trial190 2026-02-22T22:29:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:29' 2026-02-22T22:29:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:29:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:29:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:29:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}} 2026-02-22T22:29:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.2 on trial190' ']' 2026-02-22T22:29:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.2 on trial190' 2026-02-22T22:29:38.678 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.2 on trial190 2026-02-22T22:29:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:29' 2026-02-22T22:29:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:29:29' 2026-02-22T22:29:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:29:29' 2026-02-22T22:29:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:38.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGWARNINGWARNING1143514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaetrueOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOK4534mirror.0trial19019.2.3-689-gf90f4eaefalseWARNINGnot reporting statustest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}}44314434mirror.2trial1902026-02-22 22:29:29test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:29test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:37test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}44314434mirror.2trial1902026-02-22 22:29:29test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:29test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:29test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29' 2026-02-22T22:29:38.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:38.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:29:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:29' 2026-02-22T22:29:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:29:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:29:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:29:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:29:38.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:29:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:29:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:29:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:38.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:29:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:29:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:29:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:29:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:29:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:29:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:29:39.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:29:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:29:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}} 2026-02-22T22:29:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:29' 2026-02-22T22:29:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:29:39.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:29:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:29:39.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:29:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:29:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:29:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:29:39.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:29:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:29:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:39.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK11test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}}2026-02-22 22:29:29test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:29test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:37test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:29:29test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:29test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:29test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29' 2026-02-22T22:29:39.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:39.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:29:39.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:39.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:29:39.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:29:39.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:39.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:29:39.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:29:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:29:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:29:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:29:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:29:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:29:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:29:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:29:39.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:29:39.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:40.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:29:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:2 mirror test11 2026-02-22T22:29:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:2 mirror test11 started 2026-02-22T22:29:40.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:40.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:40.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:40.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:29:40.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:29:40.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:40.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:2 mirror test11 started 2026-02-22T22:29:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:29:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:29:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:29:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:2 rbd mirror status mirror/test11 2026-02-22T22:29:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:29:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:2 2026-02-22T22:29:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=2 2026-02-22T22:29:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:29:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:41.231 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:29:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test11 2026-02-22T22:29:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:29:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:29:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:29:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:29:41.236 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test11 2026-02-22T22:29:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:29:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:29:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:29:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:29:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:29:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:29:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:29:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test11 100 2026-02-22T22:29:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:29:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:29:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:29:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test11 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:29:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:29:41.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:29:41.368 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:29:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:2 cluster2 mirror test11 2026-02-22T22:29:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:41.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:29:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:2 cluster2 mirror test11 2026-02-22T22:29:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:29:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:29:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:41.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:41.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test11 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test11 ']' 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test11' 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test11 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:41.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test11 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:41.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:41.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:41.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:41.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:41.593 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test11 2026-02-22T22:29:41.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test11 2026-02-22T22:29:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test11 '' 2026-02-22T22:29:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:41.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:41.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:41.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test11.status 2026-02-22T22:29:41.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test11.status 2026-02-22T22:29:41.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:41.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:41.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test11 2026-02-22T22:29:41.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:29:41.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:41.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:41.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test11 2026-02-22T22:29:41.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:41.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:41.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test11 '..*' 2026-02-22T22:29:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test11.status 2026-02-22T22:29:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test11.status 2026-02-22T22:29:41.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test11 2026-02-22T22:29:41.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:29:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:29:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:29:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:29:41.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:41.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:41.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:29:41.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:41.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:41.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:41.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=1 2026-02-22T22:29:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:41.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:41.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:29:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=3 2026-02-22T22:29:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:29:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:41.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test11 2026-02-22T22:29:41.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:41.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:41.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test11 ']' 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test11' 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test11 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:41.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test11 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:41.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:41.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:41.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:41.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:41.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:41.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:41.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:41.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:41.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test11 2026-02-22T22:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test11 2026-02-22T22:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test11 '' 2026-02-22T22:29:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test11.status 2026-02-22T22:29:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test11.status 2026-02-22T22:29:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test11 2026-02-22T22:29:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:29:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:42.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test11 2026-02-22T22:29:42.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test11 '..*' 2026-02-22T22:29:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test11.status 2026-02-22T22:29:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test11.status 2026-02-22T22:29:42.062 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:42.062 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test11 2026-02-22T22:29:42.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:42.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:42.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:42.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:42.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:29:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:29:42.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test11 2026-02-22T22:29:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test11 ']' 2026-02-22T22:29:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test11' 2026-02-22T22:29:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test11 2026-02-22T22:29:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:42.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test11 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:42.499 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:42.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test11 2026-02-22T22:29:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test11 2026-02-22T22:29:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test11 '' 2026-02-22T22:29:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test11.status 2026-02-22T22:29:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test11.status 2026-02-22T22:29:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test11 2026-02-22T22:29:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:42.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test11 2026-02-22T22:29:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test11 '..*' 2026-02-22T22:29:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test11 2026-02-22T22:29:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test11.status 2026-02-22T22:29:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test11.status 2026-02-22T22:29:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test11 2026-02-22T22:29:42.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:42.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:42.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:42.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:42.655 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:42.658 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.status 2026-02-22T22:29:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:42.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:29:42.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test11 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:42.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:42.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:42.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:42.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:42.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:42.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:42.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:42.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:43.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test11 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:43.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:43.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:43.664 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test11 2026-02-22T22:29:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test11: 2026-02-22T22:29:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 538c249f-3fcb-4b38-9230-5300b8fd0ac1 2026-02-22T22:29:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:29:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.2 on trial190 2026-02-22T22:29:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:37' 2026-02-22T22:29:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:29:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:29:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:29:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:29:43.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.2 on trial190' ']' 2026-02-22T22:29:43.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.2 on trial190' 2026-02-22T22:29:43.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.2 on trial190 2026-02-22T22:29:43.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:43.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:43.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:37' 2026-02-22T22:29:43.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test11 up+replaying '2026-02-22 22:29:37' 2026-02-22T22:29:43.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:43.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:43.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:43.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:43.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:29:37' 2026-02-22T22:29:43.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:43.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:43.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:43.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1143514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaetrueOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOK45344536mirror.0trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}}44314434mirror.2trial1902026-02-22 22:29:29test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:29test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:37test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}44314434mirror.2trial1902026-02-22 22:29:29test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:29test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:29test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29' 2026-02-22T22:29:43.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:43.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test11'\'']/last_update' 2026-02-22T22:29:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:37' 2026-02-22T22:29:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test11'\'']/state' 2026-02-22T22:29:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:29:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:29:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:29:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:29:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:29:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test11 down+unknown 2026-02-22T22:29:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test11 down+unknown '' '' 2026-02-22T22:29:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:44.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test11 2026-02-22T22:29:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test11: 2026-02-22T22:29:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 538c249f-3fcb-4b38-9230-5300b8fd0ac1 2026-02-22T22:29:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:29:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:29:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:29:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:29:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:29:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:29:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:37' 2026-02-22T22:29:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:29:44.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:29:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:29:44.806 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:29:44.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:29:44.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:44.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:29:44.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:29:44.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:44.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test11 down+unknown '' 2026-02-22T22:29:44.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:29:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:44.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK11test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}}2026-02-22 22:29:29test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:29test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:37test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:29:29test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:29test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:29test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29' 2026-02-22T22:29:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test11'\'']/last_update' 2026-02-22T22:29:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test11'\'']/state' 2026-02-22T22:29:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:29:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:29:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:29:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test11 2026-02-22T22:29:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test11 2026-02-22T22:29:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:29:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test11.export 2026-02-22T22:29:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test11.export 2026-02-22T22:29:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.export 2026-02-22T22:29:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test11.export 2026-02-22T22:29:44.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test11.export 2026-02-22T22:29:44.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:44.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test11.export 2026-02-22T22:29:44.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test11.export 2026-02-22T22:29:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test11 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.export 2026-02-22T22:29:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test11 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test11.export 2026-02-22T22:29:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test11.export 2026-02-22T22:29:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test11.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test11.export 2026-02-22T22:29:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: start some more daemon instances and test replay' 2026-02-22T22:29:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:29:45.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:29:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:29:45 TEST: start some more daemon instances and test replay 2026-02-22T22:29:45.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:29:45 TEST: start some more daemon instances and test replay 2026-02-22T22:29:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:3 2026-02-22T22:29:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:29:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:29:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:29:45.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:29:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:29:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:29:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:29:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:45.246 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:4 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:29:45.273 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:29:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:29:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:45.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.4 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:5 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:45.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.5 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:29:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ start_mirror cluster1:6 2026-02-22T22:29:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:29:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:29:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:29:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:29:45.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:29:45.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:45.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:45.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:29:45.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:29:45.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:45.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd-mirror --cluster cluster1 --id mirror.6 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:29:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ image13=test13 2026-02-22T22:29:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ create_image cluster2 mirror test13 2026-02-22T22:29:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size=128 2026-02-22T22:29:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:29:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test13 2026-02-22T22:29:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test13 2026-02-22T22:29:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:29:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:29:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:2 mirror test1 2026-02-22T22:29:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:2 mirror test1 started 2026-02-22T22:29:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:29:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:29:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:46.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:2 mirror test1 started 2026-02-22T22:29:46.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:46.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:46.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:46.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:29:46.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:29:46.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:29:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:29:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:29:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:2 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=2 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:46.384 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:29:46.385 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:29:46.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:29:46.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:29:46.386 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:46.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:46.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T22:29:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:29:46.461 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:29:46.461 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:29:46.461 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:29:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:29:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:29:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:29:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:29:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:29:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:29:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:29:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:29:46.501 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:29:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:29:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:29:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:2 cluster2 mirror test1 2026-02-22T22:29:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:29:46.528 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:2 cluster2 mirror test1 2026-02-22T22:29:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:29:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:29:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:46.529 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:46.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:46.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:46.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:46.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:46.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:46.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:46.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:46.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:46.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:46.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:46.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:46.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:46.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:46.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:46.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:46.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:46.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:46.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:46.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:29:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:46.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:46.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:29:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:46.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:46.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:46.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:46.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:46.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:29:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:46.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=14, entry_tid=101' -a 'object_number=1, tag_tid=14, entry_tid=101' = 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=13, entry_tid=101' '!=' '' 2026-02-22T22:29:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:29:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=13, entry_tid=101' = '' 2026-02-22T22:29:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:29:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:46.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=14 2026-02-22T22:29:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:46.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=13 2026-02-22T22:29:46.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:29:46.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:46.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:46.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:29:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:46.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:29:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 14 = 13 -a 101 -le 101 2026-02-22T22:29:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:46.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:47.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:47.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:47.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:47.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:47.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:29:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:47.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:29:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:47.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:47.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:47.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:47.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:47.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:47.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:47.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:47.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:47.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:47.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:47.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:47.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:29:47.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:47.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=14, entry_tid=101' -a 'object_number=1, tag_tid=14, entry_tid=101' = 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=13, entry_tid=101' '!=' 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:29:47.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:47.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:47.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:47.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:47.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:47.659 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:47.662 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:47.663 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:47.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:47.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:47.738 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:47.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:47.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:47.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:47.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:47.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:29:47.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:29:47.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:47.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:47.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:47.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:47.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:47.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:29:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T22:29:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=14, entry_tid=101' -a 'object_number=1, tag_tid=14, entry_tid=101' = 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=13, entry_tid=101' '!=' 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T22:29:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:29:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:29:48.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:29:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:29:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:48.619 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:48.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:29:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:48.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:48.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:29:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:29:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:29:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:48.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:48.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:48.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:48.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:48.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:48.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:29:48.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:29:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:48.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:29:48.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:29:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:29:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:29:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:29:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:29:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:29:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:29:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:29:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:29:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:29:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=14, entry_tid=101' -a 'object_number=1, tag_tid=14, entry_tid=101' = 'object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:29:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:29:48.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:49.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:49.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:49.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:49.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:49.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:49.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:49.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:49.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:49.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:49.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:29:49.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:29:49.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:29:49.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}} 2026-02-22T22:29:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.2 on trial190 2026-02-22T22:29:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:29' 2026-02-22T22:29:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:29:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:29:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:29:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}} 2026-02-22T22:29:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.2 on trial190' ']' 2026-02-22T22:29:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.2 on trial190' 2026-02-22T22:29:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.2 on trial190 2026-02-22T22:29:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:29' 2026-02-22T22:29:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:29:29' 2026-02-22T22:29:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:29:29' 2026-02-22T22:29:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:49.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:49.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGWARNINGWARNING1243514354mirror.0trial19019.2.3-689-gf90f4eaetrueOK44314434mirror.2trial19019.2.3-689-gf90f4eaetrueOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOK45344536mirror.0trial19019.2.3-689-gf90f4eaefalseOK4560mirror.3trial19019.2.3-689-gf90f4eaefalseWARNINGnot reporting status4564mirror.4trial19019.2.3-689-gf90f4eaefalseWARNINGnot reporting status4571mirror.5trial19019.2.3-689-gf90f4eaefalseWARNINGnot reporting status4577mirror.6trial19019.2.3-689-gf90f4eaefalseWARNINGnot reporting statustest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}}44314434mirror.2trial1902026-02-22 22:29:29test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:29test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:37test135016d2e9-08b5-49f5-b76a-6e2480467b80up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:48test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}44314434mirror.2trial1902026-02-22 22:29:29test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:29test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:29test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29' 2026-02-22T22:29:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:49.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:29:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:29' 2026-02-22T22:29:49.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:29:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:29:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:29:49.887 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:29:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:29:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:29:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:29:49.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:49.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:50.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:29:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:50.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:29:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:29:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:29:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:29:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:29:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:29:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:29:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:29:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}} 2026-02-22T22:29:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:29' 2026-02-22T22:29:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:29:50.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:29:50.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:29:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:29:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:29:50.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:50.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:29:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:29:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:50.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:29:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:50.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:29:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:50.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:50.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK12test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}}2026-02-22 22:29:29test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:29test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:37test135016d2e9-08b5-49f5-b76a-6e2480467b80down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:48test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:29:29test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:29test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:29test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29' 2026-02-22T22:29:50.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:50.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:29:50.982 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:50.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:29:50.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:29:50.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:50.985 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:29:50.986 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:29:50.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:29:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:29:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:29:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:29:50.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:50.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:29:50.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:29:50.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:29:50.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:50.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:50.992 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:29:51.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:51.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:51.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:51.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:29:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:29:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:2 mirror test13 2026-02-22T22:29:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:2 mirror test13 started 2026-02-22T22:29:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:29:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:29:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:2 mirror test13 started 2026-02-22T22:29:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:29:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:29:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:29:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:2 rbd mirror status mirror/test13 2026-02-22T22:29:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:29:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:2 2026-02-22T22:29:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=2 2026-02-22T22:29:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:29:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:52.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test13 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:52.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:29:52.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test13 2026-02-22T22:29:52.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:29:52.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:29:52.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:29:52.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:29:52.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:29:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:29:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:29:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test13 100 2026-02-22T22:29:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:29:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:29:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:29:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test13 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:29:52.462 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:29:52.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:29:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:2 cluster2 mirror test13 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:2 cluster2 mirror test13 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:52.486 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test13 2026-02-22T22:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test13 ']' 2026-02-22T22:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test13' 2026-02-22T22:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test13 2026-02-22T22:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test13 2026-02-22T22:29:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:52.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:52.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:52.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test13 2026-02-22T22:29:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test13 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test13 '' 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test13.status 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test13.status 2026-02-22T22:29:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:52.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:52.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test13 2026-02-22T22:29:52.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:29:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test13 2026-02-22T22:29:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test13 '..*' 2026-02-22T22:29:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test13.status 2026-02-22T22:29:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test13.status 2026-02-22T22:29:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test13 2026-02-22T22:29:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:29:52.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:52.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:52.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:29:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:29:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:29:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:52.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:52.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:29:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:52.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=1 2026-02-22T22:29:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:52.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:29:52.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:52.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:52.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=3 2026-02-22T22:29:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:29:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test13 2026-02-22T22:29:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test13 ']' 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test13' 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test13 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test13 2026-02-22T22:29:53.077 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:53.078 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:53.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:53.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:53.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test13 2026-02-22T22:29:53.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:53.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test13 2026-02-22T22:29:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test13 '' 2026-02-22T22:29:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test13.status 2026-02-22T22:29:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test13.status 2026-02-22T22:29:53.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:53.153 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.153 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test13 2026-02-22T22:29:53.153 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=57], [object_number=0, tag_tid=2, entry_tid=56], [object_number=3, tag_tid=2, entry_tid=55], [object_number=2, tag_tid=2, entry_tid=54]]], state=connected] 2026-02-22T22:29:53.181 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.182 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test13 2026-02-22T22:29:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test13 '..*' 2026-02-22T22:29:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test13.status 2026-02-22T22:29:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test13.status 2026-02-22T22:29:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:53.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test13 2026-02-22T22:29:53.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=57], [object_number=0, tag_tid=2, entry_tid=56], [object_number=3, tag_tid=2, entry_tid=55], [object_number=2, tag_tid=2, entry_tid=54]]], state=connected] 2026-02-22T22:29:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=57' 2026-02-22T22:29:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=57' 2026-02-22T22:29:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=57' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:29:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=57' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:29:53.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=57' 2026-02-22T22:29:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=2 2026-02-22T22:29:53.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=57' 2026-02-22T22:29:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:29:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=2 2026-02-22T22:29:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:29:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=57' 2026-02-22T22:29:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:29:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:29:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=57 2026-02-22T22:29:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 2 = 2 -a 101 -le 57 2026-02-22T22:29:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:29:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:29:53.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:29:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test13 2026-02-22T22:29:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:29:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test13 ']' 2026-02-22T22:29:53.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test13' 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test13 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test13 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test13 2026-02-22T22:29:53.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test13 2026-02-22T22:29:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test13 '' 2026-02-22T22:29:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:29:53.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test13.status 2026-02-22T22:29:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test13.status 2026-02-22T22:29:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:53.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test13 2026-02-22T22:29:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:53.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test13 2026-02-22T22:29:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test13 '..*' 2026-02-22T22:29:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:29:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:29:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test13 2026-02-22T22:29:53.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:29:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test13.status 2026-02-22T22:29:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test13.status 2026-02-22T22:29:53.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:53.545 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.545 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test13 2026-02-22T22:29:53.545 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:29:53.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:29:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:29:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:53.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:29:53.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.status 2026-02-22T22:29:53.585 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:53.585 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:29:53.585 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:53.585 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:29:53.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test13 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:53.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test13 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:29:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:54.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:29:54.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:29:54.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:54.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:54.588 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test13 2026-02-22T22:29:54.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test13: 2026-02-22T22:29:54.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 5016d2e9-08b5-49f5-b76a-6e2480467b80 2026-02-22T22:29:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:29:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.2 on trial190 2026-02-22T22:29:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:48' 2026-02-22T22:29:54.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:29:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:29:54.624 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:29:54.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:29:54.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.2 on trial190' ']' 2026-02-22T22:29:54.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.2 on trial190' 2026-02-22T22:29:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.2 on trial190 2026-02-22T22:29:54.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:54.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:48' 2026-02-22T22:29:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test13 up+replaying '2026-02-22 22:29:48' 2026-02-22T22:29:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:29:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:29:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:29:48' 2026-02-22T22:29:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:54.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:54.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1244314434mirror.2trial19019.2.3-689-gf90f4eaetrueOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOK45344536mirror.0trial19019.2.3-689-gf90f4eaefalseOK45604567mirror.3trial19019.2.3-689-gf90f4eaefalseOK45644574mirror.4trial19019.2.3-689-gf90f4eaefalseOK45714580mirror.5trial19019.2.3-689-gf90f4eaefalseOK45774584mirror.6trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}}44314434mirror.2trial1902026-02-22 22:29:29test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:29test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:37test135016d2e9-08b5-49f5-b76a-6e2480467b80up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}44314434mirror.2trial1902026-02-22 22:29:48test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}44314434mirror.2trial1902026-02-22 22:29:29test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:29test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:29:29test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:29:29' 2026-02-22T22:29:54.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:54.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test13'\'']/last_update' 2026-02-22T22:29:54.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:29:48' 2026-02-22T22:29:54.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test13'\'']/state' 2026-02-22T22:29:54.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:29:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:29:54.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:29:54.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:29:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:29:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test13 down+unknown 2026-02-22T22:29:54.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:54.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:55.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test13 down+unknown '' '' 2026-02-22T22:29:55.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:55.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:55.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:55.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:55.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:29:55.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:29:55.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:55.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:55.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test13 2026-02-22T22:29:55.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test13: 2026-02-22T22:29:55.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: 5016d2e9-08b5-49f5-b76a-6e2480467b80 2026-02-22T22:29:55.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:29:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:29:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:29:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:29:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:29:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:29:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:29:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:29:48' 2026-02-22T22:29:55.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:29:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:29:55.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:29:55.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:29:55.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:29:55.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:55.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:29:55.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:29:55.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:29:55.724 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:29:55.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:55.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test13 down+unknown '' 2026-02-22T22:29:55.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:29:55.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:55.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:55.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:29:55.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:29:55.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:29:55.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:29:55.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:29:55.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK12test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":12}}2026-02-22 22:29:29test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:29test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:37test135016d2e9-08b5-49f5-b76a-6e2480467b80down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:29:48test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:29:29test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:29test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:29:29test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:29:29' 2026-02-22T22:29:55.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:29:55.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test13'\'']/last_update' 2026-02-22T22:29:55.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:29:55.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test13'\'']/state' 2026-02-22T22:29:55.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:29:55.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:29:55.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:29:55.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:29:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test13 2026-02-22T22:29:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test13 2026-02-22T22:29:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:29:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test13.export 2026-02-22T22:29:55.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test13.export 2026-02-22T22:29:55.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:55.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.export 2026-02-22T22:29:55.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test13.export 2026-02-22T22:29:55.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:29:55.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test13.export 2026-02-22T22:29:55.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test13.export 2026-02-22T22:29:55.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test13.export 2026-02-22T22:29:55.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test13 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.export 2026-02-22T22:29:55.917 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:55.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test13 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test13.export 2026-02-22T22:29:56.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:29:56.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test13.export 2026-02-22T22:29:56.122 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test13.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test13.export 2026-02-22T22:29:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: release leader and test it is acquired by secondary' 2026-02-22T22:29:56.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:29:56.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:29:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:29:56 TEST: release leader and test it is acquired by secondary 2026-02-22T22:29:56.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:29:56 TEST: release leader and test it is acquired by secondary 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ release_leader 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror leader release' 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror leader release 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:56.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:56.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:56.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:56.184 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:56.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:56.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:56.185 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror leader release 2026-02-22T22:29:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader_released 2026-02-22T22:29:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local i 2026-02-22T22:29:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 2 2026-02-22T22:29:56.247 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 10 2026-02-22T22:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in `seq 10` 2026-02-22T22:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:56.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:56.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:29:56.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:29:56.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:29:56.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:29:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:29:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:29:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:29:56.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:29:56.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:57.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:29:57.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:29:57.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:29:57.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:29:57.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:29:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:57.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:29:57.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=0 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 test2 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test1 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:29:57.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:29:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:29:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:29:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:57.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:29:58.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test1 started 2026-02-22T22:29:58.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:29:58.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:29:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:58.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:29:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:29:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:29:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:29:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:58.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:29:58.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:29:58.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:29:58.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:29:58.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:29:58.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:29:58.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:29:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:58.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:29:59.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:29:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:29:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:29:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:29:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:29:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:29:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:29:59.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:59.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:59.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:29:59.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:29:59.545 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:29:59.545 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:29:59.545 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:29:59.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:29:59.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 2 2026-02-22T22:30:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:30:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T22:30:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:30:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:01.550 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:01.551 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:30:01.552 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:01.561 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:01.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:01.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:01.562 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T22:30:01.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:30:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:30:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:30:01.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:30:01.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:30:01.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:30:01.680 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:30:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:30:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:30:01.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:30:01.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:30:01.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:01.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:01.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test1 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:30:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:01.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:01.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:01.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:01.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:01.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:01.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:01.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:01.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:01.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:01.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:01.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:30:01.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:01.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:01.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:01.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:01.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:01.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:02.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=15, entry_tid=101], [object_number=0, tag_tid=15, entry_tid=100], [object_number=3, tag_tid=15, entry_tid=99], [object_number=2, tag_tid=15, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.019 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=15, entry_tid=101' 2026-02-22T22:30:02.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=15, entry_tid=101], [object_number=0, tag_tid=15, entry_tid=100], [object_number=3, tag_tid=15, entry_tid=99], [object_number=2, tag_tid=15, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=15, entry_tid=101' -a 'object_number=1, tag_tid=15, entry_tid=101' = 'object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=14, entry_tid=101' '!=' '' 2026-02-22T22:30:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=14, entry_tid=101' = '' 2026-02-22T22:30:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=15, entry_tid=101' 2026-02-22T22:30:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=15 2026-02-22T22:30:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=14 2026-02-22T22:30:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=15, entry_tid=101' 2026-02-22T22:30:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:02.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:02.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:30:02.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:02.074 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:30:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 15 = 14 -a 101 -le 101 2026-02-22T22:30:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:30:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:30:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:30:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:02.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:30:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:02.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=15, entry_tid=101], [object_number=0, tag_tid=15, entry_tid=100], [object_number=3, tag_tid=15, entry_tid=99], [object_number=2, tag_tid=15, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.391 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=15, entry_tid=101' 2026-02-22T22:30:02.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:02.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:02.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:02.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:02.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:02.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:02.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:02.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:02.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=15, entry_tid=101], [object_number=0, tag_tid=15, entry_tid=100], [object_number=3, tag_tid=15, entry_tid=99], [object_number=2, tag_tid=15, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=15, entry_tid=101' -a 'object_number=1, tag_tid=15, entry_tid=101' = 'object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=14, entry_tid=101' '!=' 'object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:30:02.830 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:30:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:30:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:30:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:02.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:02.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:02.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:02.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:02.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:30:02.907 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:02.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:02.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=15, entry_tid=101], [object_number=0, tag_tid=15, entry_tid=100], [object_number=3, tag_tid=15, entry_tid=99], [object_number=2, tag_tid=15, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.948 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.949 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=15, entry_tid=101' 2026-02-22T22:30:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:02.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:02.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:02.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=15, entry_tid=101], [object_number=0, tag_tid=15, entry_tid=100], [object_number=3, tag_tid=15, entry_tid=99], [object_number=2, tag_tid=15, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=14, entry_tid=101], [object_number=0, tag_tid=14, entry_tid=100], [object_number=3, tag_tid=14, entry_tid=99], [object_number=2, tag_tid=14, entry_tid=98]]], state=connected] 2026-02-22T22:30:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=15, entry_tid=101' -a 'object_number=1, tag_tid=15, entry_tid=101' = 'object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=14, entry_tid=101' '!=' 'object_number=1, tag_tid=14, entry_tid=101' 2026-02-22T22:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:30:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test1 2026-02-22T22:30:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:03.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:30:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:30:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:03.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:30:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T22:30:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:03.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:03.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:03.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:03.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T22:30:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=15, entry_tid=101], [object_number=0, tag_tid=15, entry_tid=100], [object_number=3, tag_tid=15, entry_tid=99], [object_number=2, tag_tid=15, entry_tid=98]]], state=connected] 2026-02-22T22:30:03.918 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=15, entry_tid=101], [object_number=0, tag_tid=15, entry_tid=100], [object_number=3, tag_tid=15, entry_tid=99], [object_number=2, tag_tid=15, entry_tid=98]]], state=connected] 2026-02-22T22:30:03.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:03.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=15, entry_tid=101' 2026-02-22T22:30:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:03.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:03.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:03.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:03.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:03.926 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:03.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:04.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:04.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:04.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=15, entry_tid=101], [object_number=0, tag_tid=15, entry_tid=100], [object_number=3, tag_tid=15, entry_tid=99], [object_number=2, tag_tid=15, entry_tid=98]]], state=connected] 2026-02-22T22:30:04.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=15, entry_tid=101], [object_number=0, tag_tid=15, entry_tid=100], [object_number=3, tag_tid=15, entry_tid=99], [object_number=2, tag_tid=15, entry_tid=98]]], state=connected] 2026-02-22T22:30:04.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=15, entry_tid=101' 2026-02-22T22:30:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=15, entry_tid=101' -a 'object_number=1, tag_tid=15, entry_tid=101' = 'object_number=1, tag_tid=15, entry_tid=101' 2026-02-22T22:30:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:04.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:30:04.172 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:30:04.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:04.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:04.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:04.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:04.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:04.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:30:04.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:04.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:30:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:30:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:05.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:30:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}} 2026-02-22T22:30:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:30:05.209 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:04' 2026-02-22T22:30:05.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:30:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:30:05.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:30:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}} 2026-02-22T22:30:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:30:05.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:30:05.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:30:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:05.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:04' 2026-02-22T22:30:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:30:04' 2026-02-22T22:30:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:30:04' 2026-02-22T22:30:05.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:05.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:05.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1244314434mirror.2trial19019.2.3-689-gf90f4eaefalseOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOK45344536mirror.0trial19019.2.3-689-gf90f4eaetrueOK45604567mirror.3trial19019.2.3-689-gf90f4eaefalseOK45644574mirror.4trial19019.2.3-689-gf90f4eaefalseOK45714580mirror.5trial19019.2.3-689-gf90f4eaefalseOK45774584mirror.6trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}}45344536mirror.0trial1902026-02-22 22:30:04test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test135016d2e9-08b5-49f5-b76a-6e2480467b80up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}45344536mirror.0trial1902026-02-22 22:30:04test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45344536mirror.0trial1902026-02-22 22:30:04test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45344536mirror.0trial1902026-02-22 22:30:04test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04' 2026-02-22T22:30:05.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:05.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:05.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:04' 2026-02-22T22:30:05.272 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:30:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:30:05.276 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:30:05.277 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:30:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:30:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:30:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:05.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:05.279 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:30:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:30:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:30:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:30:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:30:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:30:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:30:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}} 2026-02-22T22:30:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:04' 2026-02-22T22:30:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:30:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:30:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:30:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:30:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:30:06.315 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:06.315 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:30:06.315 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:30:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:06.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:30:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:30:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:06.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK12test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}}2026-02-22 22:30:04test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test135016d2e9-08b5-49f5-b76a-6e2480467b80down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:30:04test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:04test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:04test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04' 2026-02-22T22:30:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:06.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:30:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:30:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:30:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:30:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:30:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:30:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:30:06.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:06.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:30:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:30:06.379 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:06.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:06.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:06.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:06.636 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:30:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:0 mirror test2 2026-02-22T22:30:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:0 mirror test2 started 2026-02-22T22:30:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:30:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:30:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:0 mirror test2 started 2026-02-22T22:30:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:30:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:30:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:30:07.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:0 rbd mirror status mirror/test2 2026-02-22T22:30:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:30:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:0 2026-02-22T22:30:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=0 2026-02-22T22:30:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:30:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:07.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:30:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:30:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test2 2026-02-22T22:30:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:30:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:07.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:30:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:07.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test2 2026-02-22T22:30:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:30:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:30:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:30:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:30:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:30:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:30:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:30:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T22:30:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:30:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:30:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:30:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:30:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:30:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:30:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:30:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:0 cluster2 mirror test2 2026-02-22T22:30:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:30:07.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:0 cluster2 mirror test2 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:0 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:30:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test2 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:08.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:08.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:30:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:30:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:08.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:08.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:08.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:08.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test2 2026-02-22T22:30:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:30:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:30:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:30:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:30:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:30:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:30:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:30:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:30:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:30:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:30:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:30:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:30:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:08.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:30:08.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:30:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:30:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=5, entry_tid=101' '!=' '' 2026-02-22T22:30:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=5, entry_tid=101' = '' 2026-02-22T22:30:08.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:08.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:30:08.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:08.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:08.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=6 2026-02-22T22:30:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:08.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=5 2026-02-22T22:30:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:30:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:08.266 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:30:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:08.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:30:08.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 6 = 5 -a 101 -le 101 2026-02-22T22:30:08.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:08.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:08.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test2 2026-02-22T22:30:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:30:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:08.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:08.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:08.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:08.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:08.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:08.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:08.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test2 2026-02-22T22:30:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:30:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:30:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:30:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:30:08.544 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:08.545 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:30:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:30:08.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:30:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:30:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:30:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:30:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:30:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:30:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:30:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:08.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:08.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:08.606 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:30:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:30:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=5, entry_tid=101' '!=' 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test2 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:30:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:30:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:09.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:09.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:09.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:09.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test2 2026-02-22T22:30:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:09.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:30:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:30:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:09.084 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:30:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:30:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:09.085 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:30:09.086 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:30:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:30:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:09.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:30:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:30:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:30:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:09.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:30:09.118 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:30:09.118 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:30:09.120 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:30:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:30:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:09.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:09.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=5, entry_tid=101' '!=' 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:30:09.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:09.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:30:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:0 mirror test2 2026-02-22T22:30:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:09.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test2 ']' 2026-02-22T22:30:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:30:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:30:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:0 2026-02-22T22:30:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:09.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 '!=' cluster1:0 ']' 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:09.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:09.958 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:09.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:09.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:09.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:09.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test2 2026-02-22T22:30:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:30:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:30:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:10.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:30:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:30:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:30:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:10.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:10.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:10.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:10.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:30:10.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:30:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:10.070 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test2 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:30:10.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:30:10.073 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:10.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:30:10.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T22:30:10.112 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.status 2026-02-22T22:30:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:30:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T22:30:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:10.114 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:30:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test2 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:30:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:30:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:11.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test2 up+replaying primary_position 'mirror.0 on trial190' 2026-02-22T22:30:11.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:11.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:11.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:11.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:11.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:11.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.0 on trial190' 2026-02-22T22:30:11.116 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test2 2026-02-22T22:30:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test2: 2026-02-22T22:30:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0 2026-02-22T22:30:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T22:30:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.0 on trial190 2026-02-22T22:30:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:04' 2026-02-22T22:30:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:30:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:30:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:30:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T22:30:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.0 on trial190' ']' 2026-02-22T22:30:11.158 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.0 on trial190' 2026-02-22T22:30:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.0 on trial190 2026-02-22T22:30:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:11.159 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:04' 2026-02-22T22:30:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test2 up+replaying '2026-02-22 22:30:04' 2026-02-22T22:30:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:11.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:30:04' 2026-02-22T22:30:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:11.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:11.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1244314434mirror.2trial19019.2.3-689-gf90f4eaefalseOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOK45344536mirror.0trial19019.2.3-689-gf90f4eaetrueOK45604567mirror.3trial19019.2.3-689-gf90f4eaefalseOK45644574mirror.4trial19019.2.3-689-gf90f4eaefalseOK45714580mirror.5trial19019.2.3-689-gf90f4eaefalseOK45774584mirror.6trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}}45344536mirror.0trial1902026-02-22 22:30:04test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test135016d2e9-08b5-49f5-b76a-6e2480467b80up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}45344536mirror.0trial1902026-02-22 22:30:04test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45344536mirror.0trial1902026-02-22 22:30:04test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45344536mirror.0trial1902026-02-22 22:30:04test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45344536mirror.0trial1902026-02-22 22:30:04' 2026-02-22T22:30:11.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:11.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/last_update' 2026-02-22T22:30:11.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:04' 2026-02-22T22:30:11.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/state' 2026-02-22T22:30:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:30:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:30:11.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:30:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test2 down+unknown 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test2 down+unknown '' '' 2026-02-22T22:30:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:12.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test2 2026-02-22T22:30:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test2: 2026-02-22T22:30:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0 2026-02-22T22:30:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:30:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:30:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:30:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:30:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:30:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:12.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T22:30:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:04' 2026-02-22T22:30:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:30:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:30:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:30:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:30:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:30:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:30:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:30:12.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:12.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test2 down+unknown '' 2026-02-22T22:30:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:30:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK12test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}}2026-02-22 22:30:04test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test135016d2e9-08b5-49f5-b76a-6e2480467b80down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}2026-02-22 22:30:04test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:04test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:04test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:04' 2026-02-22T22:30:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/last_update' 2026-02-22T22:30:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:12.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test2'\'']/state' 2026-02-22T22:30:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:30:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:30:12.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:30:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test2 2026-02-22T22:30:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test2 2026-02-22T22:30:12.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:30:12.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test2.export 2026-02-22T22:30:12.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test2.export 2026-02-22T22:30:12.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:12.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export 2026-02-22T22:30:12.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test2.export 2026-02-22T22:30:12.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test2.export 2026-02-22T22:30:12.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:30:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:30:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test2 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export 2026-02-22T22:30:12.454 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test2 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:30:12.592 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:12.596 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:30:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test2.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test2.export 2026-02-22T22:30:12.692 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:12.692 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ testlog 'TEST: in loop: stop leader and test replay' 2026-02-22T22:30:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ tee -a /tmp/tmp.ek6CoRRxsq/rbd-mirror.test.log 2026-02-22T22:30:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ date '+%F %T' 2026-02-22T22:30:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo 2026-02-22 22:30:12 TEST: in loop: stop leader and test replay 2026-02-22T22:30:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:2026-02-22 22:30:12 TEST: in loop: stop leader and test replay 2026-02-22T22:30:12.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in 0 1 2 3 4 5 2026-02-22T22:30:12.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:0 2026-02-22T22:30:12.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:12.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:30:12.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:12.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:30:12.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:30:12.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:0 2026-02-22T22:30:12.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:30:12.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:30:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:30:12.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:30:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=35232 2026-02-22T22:30:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 35232 ']' 2026-02-22T22:30:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill 35232 2026-02-22T22:30:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:12.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:12.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=35232 '$2 == pid {print; exit 1}' 2026-02-22T22:30:12.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:ubuntu 35232 1.1 0.0 776460 56116 ? Ssl 22:29 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:30:12.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:13.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=35232 '$2 == pid {print; exit 1}' 2026-02-22T22:30:13.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:13.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:13.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=35232 '$2 == pid {print; exit 1}' 2026-02-22T22:30:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:13.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:13.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:13.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:13.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:13.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:30:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:13.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:13.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:13.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:13.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:13.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:13.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:13.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:13.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:30:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:30:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:30:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:30:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:30:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:30:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:14.798 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:30:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:30:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:30:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:14.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:14.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:14.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:30:14.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:30:14.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:14.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:14.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:14.866 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:14.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:14.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:30:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:30:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:30:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:14.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:14.931 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:14.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:14.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:14.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:14.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:14.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:30:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:30:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:30:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:30:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:30:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:30:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:30:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:15.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:30:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:30:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:15.021 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:30:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:30:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:30:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:30:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:30:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:30:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:30:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:15.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:30:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:30:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:30:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:30:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:15.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:15.090 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:30:15.153 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:30:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=4 2026-02-22T22:30:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 2026-02-22T22:30:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:30:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:30:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:4 mirror test1 2026-02-22T22:30:15.155 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:30:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:4 mirror test1 started 2026-02-22T22:30:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:30:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:30:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:30:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:4 mirror test1 started 2026-02-22T22:30:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:30:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:30:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:30:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:30:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:4 rbd mirror status mirror/test1 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:4 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=4 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 '!=' cluster1:4 ']' 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:4 rbd mirror status mirror/test1 2026-02-22T22:30:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:4 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:4 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:4 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=4 2026-02-22T22:30:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=4 2026-02-22T22:30:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:30:16.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:16.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror/test1 2026-02-22T22:30:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:30:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:30:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:30:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:30:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 '!=' cluster1:4 ']' 2026-02-22T22:30:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:4 rbd mirror status mirror/test1 2026-02-22T22:30:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:4 2026-02-22T22:30:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:4 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:4 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=4 2026-02-22T22:30:17.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=4 2026-02-22T22:30:17.230 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:30:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror/test1 2026-02-22T22:30:17.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:30:17.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:30:17.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:30:17.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:30:17.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:30:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:30:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:30:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:30:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:30:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:30:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:30:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:30:17.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:30:17.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:30:17.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 5000.09 bytes/sec: 20 MiB/s 2026-02-22T22:30:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:4 cluster2 mirror test1 2026-02-22T22:30:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:4 2026-02-22T22:30:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:30:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:4 cluster2 mirror test1 2026-02-22T22:30:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:4 2026-02-22T22:30:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:17.371 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:30:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:30:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:17.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:4 mirror test1 2026-02-22T22:30:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:30:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:4 rbd mirror flush mirror/test1 2026-02-22T22:30:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:4 2026-02-22T22:30:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 '!=' cluster1:4 ']' 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror flush mirror/test1 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:30:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:30:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:30:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:30:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:30:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:17.589 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror flush mirror/test1 2026-02-22T22:30:17.648 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=16, entry_tid=101], [object_number=0, tag_tid=16, entry_tid=100], [object_number=3, tag_tid=16, entry_tid=99], [object_number=2, tag_tid=16, entry_tid=98]]], state=connected] 2026-02-22T22:30:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=16, entry_tid=101], [object_number=0, tag_tid=16, entry_tid=100], [object_number=3, tag_tid=16, entry_tid=99], [object_number=2, tag_tid=16, entry_tid=98]]], state=connected] 2026-02-22T22:30:17.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:17.683 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:17.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:17.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:17.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:17.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:17.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:17.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:17.722 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=16, entry_tid=101], [object_number=0, tag_tid=16, entry_tid=100], [object_number=3, tag_tid=16, entry_tid=99], [object_number=2, tag_tid=16, entry_tid=98]]], state=connected] 2026-02-22T22:30:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=16, entry_tid=101], [object_number=0, tag_tid=16, entry_tid=100], [object_number=3, tag_tid=16, entry_tid=99], [object_number=2, tag_tid=16, entry_tid=98]]], state=connected] 2026-02-22T22:30:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=16, entry_tid=101' -a 'object_number=1, tag_tid=16, entry_tid=101' = 'object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:30:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.4 on trial190' 2026-02-22T22:30:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.4 on trial190' 2026-02-22T22:30:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:18.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.4 on trial190' 2026-02-22T22:30:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.4 on trial190' 2026-02-22T22:30:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:18.731 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:18.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:30:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}} 2026-02-22T22:30:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.4 on trial190 2026-02-22T22:30:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:16' 2026-02-22T22:30:18.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:30:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:30:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:30:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}} 2026-02-22T22:30:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.4 on trial190' ']' 2026-02-22T22:30:18.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.4 on trial190' 2026-02-22T22:30:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.4 on trial190 2026-02-22T22:30:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:18.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:16' 2026-02-22T22:30:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:30:16' 2026-02-22T22:30:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:30:16' 2026-02-22T22:30:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:18.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1244314434mirror.2trial19019.2.3-689-gf90f4eaefalseOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOK45604567mirror.3trial19019.2.3-689-gf90f4eaefalseOK45644574mirror.4trial19019.2.3-689-gf90f4eaetrueOK45714580mirror.5trial19019.2.3-689-gf90f4eaefalseOK45774584mirror.6trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}}45644574mirror.4trial1902026-02-22 22:30:16test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45644574mirror.4trial1902026-02-22 22:30:16test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45644574mirror.4trial1902026-02-22 22:30:16test135016d2e9-08b5-49f5-b76a-6e2480467b80up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45644574mirror.4trial1902026-02-22 22:30:16test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}45644574mirror.4trial1902026-02-22 22:30:16test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45644574mirror.4trial1902026-02-22 22:30:16test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45644574mirror.4trial1902026-02-22 22:30:16test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45644574mirror.4trial1902026-02-22 22:30:16test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45644574mirror.4trial1902026-02-22 22:30:16test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45644574mirror.4trial1902026-02-22 22:30:16test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45644574mirror.4trial1902026-02-22 22:30:16test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45644574mirror.4trial1902026-02-22 22:30:16' 2026-02-22T22:30:18.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:18.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:18.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:16' 2026-02-22T22:30:18.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:30:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:30:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:30:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:30:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:30:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:30:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:19.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:30:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:19.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:30:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:30:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:30:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:30:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:30:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}} 2026-02-22T22:30:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:16' 2026-02-22T22:30:19.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:30:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:30:19.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:30:19.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:30:19.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:30:19.876 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:19.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:30:19.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:30:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:30:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:19.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:30:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:19.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK12test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":15}}2026-02-22 22:30:16test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:16test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:16test135016d2e9-08b5-49f5-b76a-6e2480467b80down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:16test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:30:16test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:16test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:16test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:16test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:16test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:16test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:16test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:16' 2026-02-22T22:30:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:19.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:30:19.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:19.936 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:30:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:30:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:30:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:30:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:30:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:30:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:30:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:30:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:20.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:20.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in 0 1 2 3 4 5 2026-02-22T22:30:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:4 2026-02-22T22:30:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:30:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:30:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:4 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:4 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:4 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:30:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=4 2026-02-22T22:30:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=4 2026-02-22T22:30:20.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.4 'pid file' 2026-02-22T22:30:20.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.pid 2026-02-22T22:30:20.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.pid 2026-02-22T22:30:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=36434 2026-02-22T22:30:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 36434 ']' 2026-02-22T22:30:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill 36434 2026-02-22T22:30:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:20.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36434 '$2 == pid {print; exit 1}' 2026-02-22T22:30:20.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:ubuntu 36434 0.8 0.0 558448 63696 ? Ssl 22:29 0:00 rbd-mirror --cluster cluster1 --id mirror.4 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:30:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:20.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36434 '$2 == pid {print; exit 1}' 2026-02-22T22:30:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36434 '$2 == pid {print; exit 1}' 2026-02-22T22:30:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:30:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:30:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:30:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:30:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:30:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:30:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:21.395 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster2 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:30:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:21.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:21.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:4 2026-02-22T22:30:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:4 2026-02-22T22:30:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:30:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:30:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:21.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:21.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:21.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:30:21.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:21.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'pid file' 2026-02-22T22:30:21.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.pid 2026-02-22T22:30:21.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.pid 2026-02-22T22:30:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:30:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:30:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:30:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:22.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:30:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:30:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:22.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:22.427 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:22.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:30:22.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:30:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:30:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:30:22.494 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:22.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:30:22.570 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:30:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:30:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:30:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:22.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:30:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:30:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:30:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:30:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:30:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:22.639 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:30:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:30:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:30:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:30:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:22.649 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:30:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:30:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:30:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:30:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:30:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:30:22.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:30:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:30:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:30:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:30:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:30:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:30:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:30:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:30:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:30:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:30:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:22.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:30:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:30:22.861 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:30:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=5 2026-02-22T22:30:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 2026-02-22T22:30:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:30:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:30:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:5 mirror test1 2026-02-22T22:30:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:5 mirror test1 started 2026-02-22T22:30:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:30:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:30:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:5 mirror test1 started 2026-02-22T22:30:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:23.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:30:23.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:30:23.864 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:30:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:5 rbd mirror status mirror/test1 2026-02-22T22:30:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:30:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:5 2026-02-22T22:30:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=5 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 '!=' cluster1:5 ']' 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:5 rbd mirror status mirror/test1 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:5 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:23.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:5 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:23.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:5 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=5 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=5 2026-02-22T22:30:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:30:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:23.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror/test1 2026-02-22T22:30:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:30:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:30:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:30:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:30:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 '!=' cluster1:5 ']' 2026-02-22T22:30:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:5 rbd mirror status mirror/test1 2026-02-22T22:30:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:5 2026-02-22T22:30:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:30:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:5 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:5 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:24.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:24.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:24.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:24.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:24.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:24.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=5 2026-02-22T22:30:24.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=5 2026-02-22T22:30:24.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:30:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:24.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror/test1 2026-02-22T22:30:25.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:30:25.014 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:30:25.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:30:25.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:30:25.016 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:30:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:30:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:30:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:30:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:30:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:30:25.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:30:25.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:30:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:30:25.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:30:25.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:30:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:5 cluster2 mirror test1 2026-02-22T22:30:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:5 2026-02-22T22:30:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:30:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:5 cluster2 mirror test1 2026-02-22T22:30:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:5 2026-02-22T22:30:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:25.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:30:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:30:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:25.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:5 mirror test1 2026-02-22T22:30:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:5 rbd mirror flush mirror/test1 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:5 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 '!=' cluster1:5 ']' 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror flush mirror/test1 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:30:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:30:25.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:25.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:25.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:25.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror flush mirror/test1 2026-02-22T22:30:25.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:25.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:25.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:25.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:25.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:25.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=16, entry_tid=101], [object_number=0, tag_tid=16, entry_tid=100], [object_number=3, tag_tid=16, entry_tid=99], [object_number=2, tag_tid=16, entry_tid=98]]], state=connected] 2026-02-22T22:30:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=16, entry_tid=101], [object_number=0, tag_tid=16, entry_tid=100], [object_number=3, tag_tid=16, entry_tid=99], [object_number=2, tag_tid=16, entry_tid=98]]], state=connected] 2026-02-22T22:30:25.451 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=17, entry_tid=101' -a 'object_number=1, tag_tid=17, entry_tid=101' = 'object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=16, entry_tid=101' '!=' '' 2026-02-22T22:30:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=16, entry_tid=101' = '' 2026-02-22T22:30:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=17 2026-02-22T22:30:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=16 2026-02-22T22:30:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:30:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:30:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 17 = 16 -a 101 -le 101 2026-02-22T22:30:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:25.665 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:5 mirror test1 2026-02-22T22:30:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:5 rbd mirror flush mirror/test1 2026-02-22T22:30:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:5 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 '!=' cluster1:5 ']' 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror flush mirror/test1 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:30:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:25.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:30:25.676 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:25.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:25.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:25.677 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror flush mirror/test1 2026-02-22T22:30:25.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:25.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:25.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:25.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:25.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:25.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:25.746 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:25.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=16, entry_tid=101], [object_number=0, tag_tid=16, entry_tid=100], [object_number=3, tag_tid=16, entry_tid=99], [object_number=2, tag_tid=16, entry_tid=98]]], state=connected] 2026-02-22T22:30:25.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.784 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:25.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:25.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:25.787 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:25.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=16, entry_tid=101], [object_number=0, tag_tid=16, entry_tid=100], [object_number=3, tag_tid=16, entry_tid=99], [object_number=2, tag_tid=16, entry_tid=98]]], state=connected] 2026-02-22T22:30:25.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:25.824 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:25.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=17, entry_tid=101' -a 'object_number=1, tag_tid=17, entry_tid=101' = 'object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:25.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=16, entry_tid=101' '!=' 'object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:25.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:25.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:5 mirror test1 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:5 rbd mirror flush mirror/test1 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:5 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:26.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 '!=' cluster1:5 ']' 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror flush mirror/test1 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:30:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:26.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:26.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:30:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror flush mirror/test1 2026-02-22T22:30:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:26.345 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:26.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=16, entry_tid=101], [object_number=0, tag_tid=16, entry_tid=100], [object_number=3, tag_tid=16, entry_tid=99], [object_number=2, tag_tid=16, entry_tid=98]]], state=connected] 2026-02-22T22:30:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:26.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:26.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:26.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:26.353 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:26.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:26.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:26.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:26.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:26.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:26.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:26.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:26.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=16, entry_tid=101], [object_number=0, tag_tid=16, entry_tid=100], [object_number=3, tag_tid=16, entry_tid=99], [object_number=2, tag_tid=16, entry_tid=98]]], state=connected] 2026-02-22T22:30:26.392 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:26.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:26.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=17, entry_tid=101' -a 'object_number=1, tag_tid=17, entry_tid=101' = 'object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:26.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=16, entry_tid=101' '!=' 'object_number=1, tag_tid=16, entry_tid=101' 2026-02-22T22:30:26.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:26.394 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:5 mirror test1 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:5 rbd mirror flush mirror/test1 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:5 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 '!=' cluster1:5 ']' 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror flush mirror/test1 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:30:27.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:30:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror flush mirror/test1 2026-02-22T22:30:27.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:27.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:27.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:27.310 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:27.310 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:27.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:27.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:27.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:27.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:27.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:27.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:27.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:27.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:27.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:27.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:27.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:27.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:27.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:27.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:27.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:27.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=17, entry_tid=101' -a 'object_number=1, tag_tid=17, entry_tid=101' = 'object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:30:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.5 on trial190' 2026-02-22T22:30:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.5 on trial190' 2026-02-22T22:30:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:28.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.5 on trial190' 2026-02-22T22:30:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.5 on trial190' 2026-02-22T22:30:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:28.362 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:30:28.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:28.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:28.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:28.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":16},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":16}} 2026-02-22T22:30:28.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.5 on trial190 2026-02-22T22:30:28.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:24' 2026-02-22T22:30:28.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:30:28.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:30:28.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:30:28.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":16},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":16}} 2026-02-22T22:30:28.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.5 on trial190' ']' 2026-02-22T22:30:28.400 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.5 on trial190' 2026-02-22T22:30:28.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.5 on trial190 2026-02-22T22:30:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:28.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:28.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:24' 2026-02-22T22:30:28.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:30:24' 2026-02-22T22:30:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:30:24' 2026-02-22T22:30:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:28.467 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1244314434mirror.2trial19019.2.3-689-gf90f4eaefalseOK44904492mirror.1trial19019.2.3-689-gf90f4eaefalseOK45604567mirror.3trial19019.2.3-689-gf90f4eaefalseOK45714580mirror.5trial19019.2.3-689-gf90f4eaetrueOK45774584mirror.6trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":16},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":16}}45714580mirror.5trial1902026-02-22 22:30:24test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45714580mirror.5trial1902026-02-22 22:30:24test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45714580mirror.5trial1902026-02-22 22:30:24test135016d2e9-08b5-49f5-b76a-6e2480467b80up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45714580mirror.5trial1902026-02-22 22:30:24test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}45714580mirror.5trial1902026-02-22 22:30:24test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45714580mirror.5trial1902026-02-22 22:30:24test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45714580mirror.5trial1902026-02-22 22:30:24test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45714580mirror.5trial1902026-02-22 22:30:24test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45714580mirror.5trial1902026-02-22 22:30:24test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45714580mirror.5trial1902026-02-22 22:30:24test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45714580mirror.5trial1902026-02-22 22:30:24test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45714580mirror.5trial1902026-02-22 22:30:24' 2026-02-22T22:30:28.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:28.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:24' 2026-02-22T22:30:28.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:30:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:30:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:30:28.476 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:30:28.477 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:29.478 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:30:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:29.479 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:30:29.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:30:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:30:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:30:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:30:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:30:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":16},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":16}} 2026-02-22T22:30:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:24' 2026-02-22T22:30:29.511 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:30:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:30:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:30:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:30:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:30:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:30:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:30:29.516 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:29.517 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:30:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:30:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK12test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":16},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":16}}2026-02-22 22:30:24test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:24test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:24test135016d2e9-08b5-49f5-b76a-6e2480467b80down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:24test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:30:24test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:24test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:24test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:24test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:24test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:24test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:24test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:24' 2026-02-22T22:30:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:29.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:29.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:29.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:29.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:30:29.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:29.572 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:30:29.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:30:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:30:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:30:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:30:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:30:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:30:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:30:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:29.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:29.578 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:29.579 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:29.709 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:29.919 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in 0 1 2 3 4 5 2026-02-22T22:30:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:5 2026-02-22T22:30:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:30:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:30:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:5 2026-02-22T22:30:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:5 2026-02-22T22:30:29.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:5 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=5 2026-02-22T22:30:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=5 2026-02-22T22:30:29.956 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.5 'pid file' 2026-02-22T22:30:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.pid 2026-02-22T22:30:29.966 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.pid 2026-02-22T22:30:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=36494 2026-02-22T22:30:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 36494 ']' 2026-02-22T22:30:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill 36494 2026-02-22T22:30:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36494 '$2 == pid {print; exit 1}' 2026-02-22T22:30:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:ubuntu 36494 0.7 0.0 777688 55460 ? Ssl 22:29 0:00 rbd-mirror --cluster cluster1 --id mirror.5 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:30:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:30.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36494 '$2 == pid {print; exit 1}' 2026-02-22T22:30:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:30.995 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36494 '$2 == pid {print; exit 1}' 2026-02-22T22:30:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:31.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:30:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:31.023 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster2 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:31.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:30:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:5 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:5 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:31.038 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'pid file' 2026-02-22T22:30:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.pid 2026-02-22T22:30:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.pid 2026-02-22T22:30:31.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:30:31.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:30:31.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:30:31.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:32.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:30:32.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:32.054 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:30:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:30:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:30:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:32.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:30:32.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:30:32.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=1 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:1 mirror test1 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:1 mirror test1 started 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:32.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:1 mirror test1 started 2026-02-22T22:30:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:30:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:30:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:30:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:30:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:30:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:1 2026-02-22T22:30:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=1 2026-02-22T22:30:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:30:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:30:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:30:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:33.196 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:30:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:30:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:33.205 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:33.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:33.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:33.206 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T22:30:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:30:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:30:33.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:33.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:30:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:30:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T22:30:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:30:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:34.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:30:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:30:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:34.274 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:34.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:34.275 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T22:30:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:30:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:30:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:30:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:30:34.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:30:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:30:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:30:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:30:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:30:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:30:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:30:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:30:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:30:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:30:34.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:30:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:1 cluster2 mirror test1 2026-02-22T22:30:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:30:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:34.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:30:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:1 cluster2 mirror test1 2026-02-22T22:30:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:1 2026-02-22T22:30:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:30:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:30:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:34.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:34.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:30:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:30:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:30:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:30:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:30:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:30:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:30:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:34.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:34.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:34.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:34.632 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:30:34.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:34.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:34.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:34.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:34.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:34.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:34.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:34.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:34.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:34.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:34.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:34.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:34.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:34.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:34.739 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:34.740 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:34.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:34.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:34.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:34.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:34.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:34.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:34.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:34.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:34.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:34.742 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:34.743 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:34.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=17, entry_tid=101], [object_number=0, tag_tid=17, entry_tid=100], [object_number=3, tag_tid=17, entry_tid=99], [object_number=2, tag_tid=17, entry_tid=98]]], state=connected] 2026-02-22T22:30:34.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=18, entry_tid=101' -a 'object_number=1, tag_tid=18, entry_tid=101' = 'object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=17, entry_tid=101' '!=' '' 2026-02-22T22:30:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=17, entry_tid=101' = '' 2026-02-22T22:30:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=18 2026-02-22T22:30:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=17 2026-02-22T22:30:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:30:34.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=17, entry_tid=101' 2026-02-22T22:30:34.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:34.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:34.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:30:34.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 18 = 17 -a 101 -le 101 2026-02-22T22:30:34.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:34.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:34.783 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:1 mirror test1 2026-02-22T22:30:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:34.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:1 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 '!=' cluster1:1 ']' 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test1 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:34.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test1 2026-02-22T22:30:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:35.097 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:35.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:35.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:35.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:35.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:35.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:35.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:35.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:35.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=18, entry_tid=101' -a 'object_number=1, tag_tid=18, entry_tid=101' = 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:35.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:35.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:30:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:30:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:30:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.1 on trial190' 2026-02-22T22:30:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.1 on trial190' 2026-02-22T22:30:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:30:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":17},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":17}} 2026-02-22T22:30:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.1 on trial190 2026-02-22T22:30:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:34' 2026-02-22T22:30:36.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:30:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:30:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:30:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":17},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":17}} 2026-02-22T22:30:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.1 on trial190' ']' 2026-02-22T22:30:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.1 on trial190' 2026-02-22T22:30:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.1 on trial190 2026-02-22T22:30:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:36.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:34' 2026-02-22T22:30:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:30:34' 2026-02-22T22:30:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:30:34' 2026-02-22T22:30:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1244314434mirror.2trial19019.2.3-689-gf90f4eaefalseOK44904492mirror.1trial19019.2.3-689-gf90f4eaetrueOK45604567mirror.3trial19019.2.3-689-gf90f4eaefalseOK45774584mirror.6trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":17},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":17}}44904492mirror.1trial1902026-02-22 22:30:34test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44904492mirror.1trial1902026-02-22 22:30:34test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44904492mirror.1trial1902026-02-22 22:30:34test135016d2e9-08b5-49f5-b76a-6e2480467b80up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44904492mirror.1trial1902026-02-22 22:30:34test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}44904492mirror.1trial1902026-02-22 22:30:34test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44904492mirror.1trial1902026-02-22 22:30:34test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44904492mirror.1trial1902026-02-22 22:30:34test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44904492mirror.1trial1902026-02-22 22:30:34test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44904492mirror.1trial1902026-02-22 22:30:34test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44904492mirror.1trial1902026-02-22 22:30:34test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44904492mirror.1trial1902026-02-22 22:30:34test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44904492mirror.1trial1902026-02-22 22:30:34' 2026-02-22T22:30:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:34' 2026-02-22T22:30:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:30:36.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:30:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:30:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:30:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:30:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:30:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:36.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:36.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:30:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:30:37.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:37.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:37.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:30:37.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:30:37.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:30:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:30:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:30:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":17},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":17}} 2026-02-22T22:30:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:34' 2026-02-22T22:30:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:30:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:30:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:30:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:30:37.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:30:37.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:37.298 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:30:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:30:37.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:37.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:30:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:30:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK12test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":17},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":17}}2026-02-22 22:30:34test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:34test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:34test135016d2e9-08b5-49f5-b76a-6e2480467b80down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:34test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:30:34test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:34test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:34test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:34test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:34test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:34test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:34test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:34' 2026-02-22T22:30:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:37.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:37.357 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:37.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:30:37.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:37.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:30:37.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:30:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:30:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:37.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:30:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:30:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:30:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:30:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:30:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:37.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:37.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:37.367 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:37.515 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:37.638 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:37.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in 0 1 2 3 4 5 2026-02-22T22:30:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:1 2026-02-22T22:30:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:30:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:30:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T22:30:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:1 2026-02-22T22:30:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:30:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:30:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:30:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:30:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:30:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:30:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:30:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:30:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:30:37.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=32590 2026-02-22T22:30:37.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 32590 ']' 2026-02-22T22:30:37.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill 32590 2026-02-22T22:30:37.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:37.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:37.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=32590 '$2 == pid {print; exit 1}' 2026-02-22T22:30:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:ubuntu 32590 0.3 0.0 560072 58820 ? Ssl 22:28 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:30:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:38.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:38.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=32590 '$2 == pid {print; exit 1}' 2026-02-22T22:30:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:38.801 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=32590 '$2 == pid {print; exit 1}' 2026-02-22T22:30:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:38.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:38.825 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T22:30:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:38.839 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:38.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:1 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:38.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:30:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:30:38.852 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:30:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:30:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:30:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:30:38.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:39.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:30:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:39.858 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:39.859 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:39.860 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:39.861 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:30:39.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:30:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:30:39.983 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:30:39.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:30:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:30:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:30:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:40.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:30:40.059 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:30:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:30:40.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:30:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:30:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 4 2026-02-22T22:30:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=4 2026-02-22T22:30:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:4 rbd mirror status mirror cluster2 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:4 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=4 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=4 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:30:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:30:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:30:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:30:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:30:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:30:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:30:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok rbd mirror status mirror cluster2 2026-02-22T22:30:40.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 5 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=5 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:5 rbd mirror status mirror cluster2 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:40.191 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:5 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=5 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=5 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:30:40.192 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:30:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:30:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:30:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:40.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:30:40.202 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok rbd mirror status mirror cluster2 2026-02-22T22:30:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:40.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:40.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:40.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 6 2026-02-22T22:30:40.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror status mirror cluster2 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:30:40.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:30:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:30:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:40.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:30:40.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:40.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:40.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:40.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror cluster2 2026-02-22T22:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:30:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=6 2026-02-22T22:30:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 2026-02-22T22:30:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:30:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:30:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:6 mirror test1 2026-02-22T22:30:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:6 mirror test1 started 2026-02-22T22:30:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:30:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:30:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:41.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:6 mirror test1 started 2026-02-22T22:30:41.334 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:41.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:41.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:41.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:30:41.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:30:41.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:30:41.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:30:41.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:6 rbd mirror status mirror/test1 2026-02-22T22:30:41.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:6 2026-02-22T22:30:41.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:41.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:41.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=6 2026-02-22T22:30:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:30:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:41.337 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:30:41.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 '!=' cluster1:6 ']' 2026-02-22T22:30:41.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:6 rbd mirror status mirror/test1 2026-02-22T22:30:41.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:6 2026-02-22T22:30:41.339 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:30:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:6 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:6 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:41.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=6 2026-02-22T22:30:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=6 2026-02-22T22:30:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:30:41.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:41.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror/test1 2026-02-22T22:30:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:30:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:30:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:30:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 '!=' cluster1:6 ']' 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:6 rbd mirror status mirror/test1 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:6 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:6 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:6 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=6 2026-02-22T22:30:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=6 2026-02-22T22:30:42.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:30:42.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:42.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:42.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:42.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror status mirror/test1 2026-02-22T22:30:42.489 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:30:42.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:30:42.490 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:30:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:30:42.491 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:30:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:30:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:30:42.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:30:42.555 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:6 cluster2 mirror test1 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:6 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:6 cluster2 mirror test1 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:6 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:30:42.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:30:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:42.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:42.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:6 mirror test1 2026-02-22T22:30:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:6 rbd mirror flush mirror/test1 2026-02-22T22:30:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:6 2026-02-22T22:30:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:42.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 '!=' cluster1:6 ']' 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror flush mirror/test1 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror flush mirror/test1 2026-02-22T22:30:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:42.848 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:42.851 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:42.851 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=19, entry_tid=101], [object_number=0, tag_tid=19, entry_tid=100], [object_number=3, tag_tid=19, entry_tid=99], [object_number=2, tag_tid=19, entry_tid=98]]], state=connected] 2026-02-22T22:30:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:42.889 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:42.891 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=19, entry_tid=101' 2026-02-22T22:30:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:42.893 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:42.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=19, entry_tid=101], [object_number=0, tag_tid=19, entry_tid=100], [object_number=3, tag_tid=19, entry_tid=99], [object_number=2, tag_tid=19, entry_tid=98]]], state=connected] 2026-02-22T22:30:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:42.933 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=19, entry_tid=101' -a 'object_number=1, tag_tid=19, entry_tid=101' = 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=18, entry_tid=101' '!=' '' 2026-02-22T22:30:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=18, entry_tid=101' = '' 2026-02-22T22:30:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=19, entry_tid=101' 2026-02-22T22:30:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:42.936 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=19 2026-02-22T22:30:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:42.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=18 2026-02-22T22:30:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=19, entry_tid=101' 2026-02-22T22:30:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:42.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:30:42.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:42.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 19 = 18 -a 101 -le 101 2026-02-22T22:30:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:42.944 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:6 mirror test1 2026-02-22T22:30:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:6 rbd mirror flush mirror/test1 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:6 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 '!=' cluster1:6 ']' 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror flush mirror/test1 2026-02-22T22:30:43.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:30:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror flush mirror/test1 2026-02-22T22:30:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=19, entry_tid=101], [object_number=0, tag_tid=19, entry_tid=100], [object_number=3, tag_tid=19, entry_tid=99], [object_number=2, tag_tid=19, entry_tid=98]]], state=connected] 2026-02-22T22:30:43.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:43.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=19, entry_tid=101' 2026-02-22T22:30:43.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:43.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:43.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:43.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:43.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:43.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=19, entry_tid=101], [object_number=0, tag_tid=19, entry_tid=100], [object_number=3, tag_tid=19, entry_tid=99], [object_number=2, tag_tid=19, entry_tid=98]]], state=connected] 2026-02-22T22:30:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=19, entry_tid=101' -a 'object_number=1, tag_tid=19, entry_tid=101' = 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=18, entry_tid=101' '!=' 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:30:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:6 mirror test1 2026-02-22T22:30:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:6 rbd mirror flush mirror/test1 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:6 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 '!=' cluster1:6 ']' 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror flush mirror/test1 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:30:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:30:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:30:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:30:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:30:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:43.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:43.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:43.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror flush mirror/test1 2026-02-22T22:30:43.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=19, entry_tid=101], [object_number=0, tag_tid=19, entry_tid=100], [object_number=3, tag_tid=19, entry_tid=99], [object_number=2, tag_tid=19, entry_tid=98]]], state=connected] 2026-02-22T22:30:43.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:43.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=19, entry_tid=101' 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:43.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:43.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:43.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:43.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:43.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:43.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:43.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=19, entry_tid=101], [object_number=0, tag_tid=19, entry_tid=100], [object_number=3, tag_tid=19, entry_tid=99], [object_number=2, tag_tid=19, entry_tid=98]]], state=connected] 2026-02-22T22:30:43.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T22:30:43.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=19, entry_tid=101' -a 'object_number=1, tag_tid=19, entry_tid=101' = 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:43.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=18, entry_tid=101' '!=' 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T22:30:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:43.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:6 mirror test1 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:6 rbd mirror flush mirror/test1 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:6 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=6 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 '!=' cluster1:6 ']' 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:6 rbd mirror flush mirror/test1 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:6 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=6 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=6 2026-02-22T22:30:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:30:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:44.657 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok rbd mirror flush mirror/test1 2026-02-22T22:30:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=19, entry_tid=101], [object_number=0, tag_tid=19, entry_tid=100], [object_number=3, tag_tid=19, entry_tid=99], [object_number=2, tag_tid=19, entry_tid=98]]], state=connected] 2026-02-22T22:30:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=19, entry_tid=101], [object_number=0, tag_tid=19, entry_tid=100], [object_number=3, tag_tid=19, entry_tid=99], [object_number=2, tag_tid=19, entry_tid=98]]], state=connected] 2026-02-22T22:30:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:44.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=19, entry_tid=101' 2026-02-22T22:30:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:44.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:44.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:44.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:44.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=19, entry_tid=101], [object_number=0, tag_tid=19, entry_tid=100], [object_number=3, tag_tid=19, entry_tid=99], [object_number=2, tag_tid=19, entry_tid=98]]], state=connected] 2026-02-22T22:30:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=19, entry_tid=101], [object_number=0, tag_tid=19, entry_tid=100], [object_number=3, tag_tid=19, entry_tid=99], [object_number=2, tag_tid=19, entry_tid=98]]], state=connected] 2026-02-22T22:30:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=19, entry_tid=101' 2026-02-22T22:30:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=19, entry_tid=101' -a 'object_number=1, tag_tid=19, entry_tid=101' = 'object_number=1, tag_tid=19, entry_tid=101' 2026-02-22T22:30:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:30:44.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.6 on trial190' 2026-02-22T22:30:44.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:44.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.6 on trial190' 2026-02-22T22:30:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:44.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.6 on trial190' 2026-02-22T22:30:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.6 on trial190' 2026-02-22T22:30:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:30:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:45.842 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19}} 2026-02-22T22:30:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.6 on trial190 2026-02-22T22:30:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:45' 2026-02-22T22:30:45.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:30:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:30:45.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:30:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19}} 2026-02-22T22:30:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.6 on trial190' ']' 2026-02-22T22:30:45.845 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.6 on trial190' 2026-02-22T22:30:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.6 on trial190 2026-02-22T22:30:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:45' 2026-02-22T22:30:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:30:45' 2026-02-22T22:30:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:30:45' 2026-02-22T22:30:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:45.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1244314434mirror.2trial19019.2.3-689-gf90f4eaefalseOK45604567mirror.3trial19019.2.3-689-gf90f4eaefalseOK45774584mirror.6trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19}}45774584mirror.6trial1902026-02-22 22:30:45test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45774584mirror.6trial1902026-02-22 22:30:45test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45774584mirror.6trial1902026-02-22 22:30:45test135016d2e9-08b5-49f5-b76a-6e2480467b80up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45774584mirror.6trial1902026-02-22 22:30:45test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}45774584mirror.6trial1902026-02-22 22:30:45test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45774584mirror.6trial1902026-02-22 22:30:45test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45774584mirror.6trial1902026-02-22 22:30:45test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45774584mirror.6trial1902026-02-22 22:30:45test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45774584mirror.6trial1902026-02-22 22:30:45test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45774584mirror.6trial1902026-02-22 22:30:45test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45774584mirror.6trial1902026-02-22 22:30:45test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45774584mirror.6trial1902026-02-22 22:30:45' 2026-02-22T22:30:45.901 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:45.902 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:45' 2026-02-22T22:30:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:30:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:30:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:30:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:46.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:30:46.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:46.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:46.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:46.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:46.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:46.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:46.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:46.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:46.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:30:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:30:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:30:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:30:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:30:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:30:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19}} 2026-02-22T22:30:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:45' 2026-02-22T22:30:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:30:46.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:30:46.947 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:30:46.948 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:30:46.948 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:30:46.948 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:46.948 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:30:46.949 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:30:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:30:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:30:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:47.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK12test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19}}2026-02-22 22:30:45test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:45test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:45test135016d2e9-08b5-49f5-b76a-6e2480467b80down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:45test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:30:45test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:45test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:45test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:45test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:45test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:45test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:45test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:45' 2026-02-22T22:30:47.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:47.000 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:47.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:30:47.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:47.007 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:30:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:30:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:30:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:30:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:30:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:30:47.010 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:30:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:30:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:47.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:47.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:47.352 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:47.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:47.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in 0 1 2 3 4 5 2026-02-22T22:30:47.397 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:6 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:6 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:6 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:6 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:47.398 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:30:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:30:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=6 2026-02-22T22:30:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=6 2026-02-22T22:30:47.399 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.6 'pid file' 2026-02-22T22:30:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.pid 2026-02-22T22:30:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.pid 2026-02-22T22:30:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=36539 2026-02-22T22:30:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 36539 ']' 2026-02-22T22:30:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill 36539 2026-02-22T22:30:47.411 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36539 '$2 == pid {print; exit 1}' 2026-02-22T22:30:47.428 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:ubuntu 36539 0.5 0.0 779624 56272 ? Ssl 22:29 0:00 rbd-mirror --cluster cluster1 --id mirror.6 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:30:47.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:47.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:48.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:48.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36539 '$2 == pid {print; exit 1}' 2026-02-22T22:30:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36539 '$2 == pid {print; exit 1}' 2026-02-22T22:30:48.458 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:48.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:30:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster2 2026-02-22T22:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:30:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:30:48.484 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:6 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:6 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:30:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'pid file' 2026-02-22T22:30:48.495 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.pid 2026-02-22T22:30:48.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.pid 2026-02-22T22:30:48.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:30:48.496 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:30:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:30:48.497 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:49.498 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:30:49.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:49.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:30:49.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:49.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:49.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:49.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:49.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:30:49.502 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:30:49.560 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:30:49.564 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:30:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:30:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:30:49.565 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:30:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:30:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:49.566 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:49.575 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:49.576 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:30:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:30:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:30:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:30:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:49.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:49.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:49.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:49.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:49.637 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:30:49.694 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:30:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=2 2026-02-22T22:30:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 2026-02-22T22:30:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:30:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:30:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:2 mirror test1 2026-02-22T22:30:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:49.696 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:2 mirror test1 started 2026-02-22T22:30:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:30:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:30:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:49.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:2 mirror test1 started 2026-02-22T22:30:50.698 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:30:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:30:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:30:50.699 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:30:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:30:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:2 2026-02-22T22:30:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=2 2026-02-22T22:30:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:30:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:50.701 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:30:50.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:30:50.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:30:50.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:30:50.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:50.704 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:50.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:50.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:30:50.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:30:50.707 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:50.712 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:50.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T22:30:50.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:30:50.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:30:50.764 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:30:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:30:51.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:30:51.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T22:30:51.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:30:51.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:51.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:51.767 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:30:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:30:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:51.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:51.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:51.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:51.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T22:30:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:30:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:30:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:30:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:30:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:30:51.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:30:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:30:51.913 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:30:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:2 cluster2 mirror test1 2026-02-22T22:30:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:30:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:30:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:2 cluster2 mirror test1 2026-02-22T22:30:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:2 2026-02-22T22:30:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:30:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:30:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:30:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:30:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:30:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:30:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:30:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:30:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:52.140 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:52.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:30:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:52.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:52.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:52.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:52.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:30:52.251 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=21], [object_number=0, tag_tid=20, entry_tid=20], [object_number=3, tag_tid=20, entry_tid=19], [object_number=2, tag_tid=20, entry_tid=18]]], state=connected] 2026-02-22T22:30:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:30:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:52.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:52.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:52.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:52.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:52.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:52.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:52.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:52.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:52.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:30:52.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=21], [object_number=0, tag_tid=20, entry_tid=20], [object_number=3, tag_tid=20, entry_tid=19], [object_number=2, tag_tid=20, entry_tid=18]]], state=connected] 2026-02-22T22:30:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=20, entry_tid=21' 2026-02-22T22:30:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=20, entry_tid=101' -a 'object_number=1, tag_tid=20, entry_tid=101' = 'object_number=1, tag_tid=20, entry_tid=21' 2026-02-22T22:30:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=20, entry_tid=21' '!=' '' 2026-02-22T22:30:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=20, entry_tid=21' = '' 2026-02-22T22:30:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=20, entry_tid=21' 2026-02-22T22:30:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:30:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=20 2026-02-22T22:30:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=20, entry_tid=21' 2026-02-22T22:30:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:30:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=20 2026-02-22T22:30:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:30:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:30:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=20, entry_tid=21' 2026-02-22T22:30:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:30:52.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:30:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=21 2026-02-22T22:30:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 20 = 20 -a 101 -le 21 2026-02-22T22:30:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:52.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:2 mirror test1 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:2 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:52.508 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 '!=' cluster1:2 ']' 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test1 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:30:52.509 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:52.510 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test1 2026-02-22T22:30:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:30:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:30:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:52.583 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:30:52.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:52.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:52.584 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:52.585 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:52.586 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:30:52.622 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:30:52.626 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.627 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:30:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:30:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:30:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:30:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:30:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:30:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:30:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:30:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:30:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:30:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:30:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:30:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:30:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:30:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:30:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:30:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=20, entry_tid=101' -a 'object_number=1, tag_tid=20, entry_tid=101' = 'object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:30:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:52.670 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:30:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:30:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:30:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:53.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.2 on trial190' 2026-02-22T22:30:53.673 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:30:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.2 on trial190' 2026-02-22T22:30:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:53.674 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:30:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19}} 2026-02-22T22:30:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.2 on trial190 2026-02-22T22:30:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:51' 2026-02-22T22:30:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:30:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:30:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:30:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19}} 2026-02-22T22:30:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.2 on trial190' ']' 2026-02-22T22:30:53.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.2 on trial190' 2026-02-22T22:30:53.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.2 on trial190 2026-02-22T22:30:53.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:53.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:51' 2026-02-22T22:30:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:30:51' 2026-02-22T22:30:53.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:30:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:30:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:30:51' 2026-02-22T22:30:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:53.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1244314434mirror.2trial19019.2.3-689-gf90f4eaetrueOK45604567mirror.3trial19019.2.3-689-gf90f4eaefalseOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19}}44314434mirror.2trial1902026-02-22 22:30:51test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:30:51test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:30:51test135016d2e9-08b5-49f5-b76a-6e2480467b80up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:30:51test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}44314434mirror.2trial1902026-02-22 22:30:51test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:30:51test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}44314434mirror.2trial1902026-02-22 22:30:51test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:30:51test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:30:51test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:30:51test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:30:51test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}44314434mirror.2trial1902026-02-22 22:30:51' 2026-02-22T22:30:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:51' 2026-02-22T22:30:53.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:30:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:30:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:30:53.779 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:30:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:30:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:30:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:53.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:53.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:30:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:54.781 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:54.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:54.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:54.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:30:54.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:30:54.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:54.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:54.782 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:30:54.811 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:30:54.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:30:54.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:30:54.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:30:54.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:30:54.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:30:54.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:30:54.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:30:54.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19}} 2026-02-22T22:30:54.812 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:51' 2026-02-22T22:30:54.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:30:54.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:30:54.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:30:54.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:30:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:30:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:54.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:30:54.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:30:54.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:30:54.818 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:30:54.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:54.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:30:54.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:54.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:54.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:54.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:30:54.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:30:54.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:30:54.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:30:54.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:30:54.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK12test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":19}}2026-02-22 22:30:51test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:51test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:51test135016d2e9-08b5-49f5-b76a-6e2480467b80down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:51test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:30:51test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:51test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:51test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:51test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:51test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:51test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:51test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:51' 2026-02-22T22:30:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:30:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:30:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:30:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:30:54.874 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:30:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:30:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:30:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:30:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:30:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:30:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:30:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:30:54.877 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:30:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:30:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:30:54.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:54.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:54.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:30:54.997 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:55.003 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:30:55.141 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:55.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:30:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for i in 0 1 2 3 4 5 2026-02-22T22:30:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:2 2026-02-22T22:30:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:30:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:2 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:30:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:30:55.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:30:55.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:30:55.245 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:30:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:30:55.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:30:55.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=29158 2026-02-22T22:30:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 29158 ']' 2026-02-22T22:30:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill 29158 2026-02-22T22:30:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=29158 '$2 == pid {print; exit 1}' 2026-02-22T22:30:55.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:ubuntu 29158 0.7 0.0 769668 64188 ? Rsl 22:27 0:01 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:30:55.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:30:55.269 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:56.270 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=29158 '$2 == pid {print; exit 1}' 2026-02-22T22:30:56.282 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:30:56.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:30:56.283 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=29158 '$2 == pid {print; exit 1}' 2026-02-22T22:30:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:30:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:30:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:56.301 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:56.302 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:56.313 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T22:30:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:30:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:30:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:56.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:56.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:2 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:56.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:56.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:56.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:56.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:30:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:30:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:30:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_leader 2026-02-22T22:30:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s instance 2026-02-22T22:30:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 2026-02-22T22:30:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:30:57.346 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 0 2026-02-22T22:30:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=0 2026-02-22T22:30:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror cluster2 2026-02-22T22:30:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:0 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=0 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=0 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:30:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:30:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:30:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:30:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:30:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:57.358 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:30:57.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror cluster2 2026-02-22T22:30:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 1 2026-02-22T22:30:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=1 2026-02-22T22:30:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:1 rbd mirror status mirror cluster2 2026-02-22T22:30:57.407 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:1 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=1 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=1 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:30:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:30:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:30:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:30:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror cluster2 2026-02-22T22:30:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 2 2026-02-22T22:30:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=2 2026-02-22T22:30:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:2 rbd mirror status mirror cluster2 2026-02-22T22:30:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:2 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=2 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=2 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:30:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:30:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:30:57.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:57.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:57.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:30:57.480 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror cluster2 2026-02-22T22:30:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-22T22:30:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ continue 2026-02-22T22:30:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:30:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ is_leader 3 2026-02-22T22:30:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:30:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool= 2026-02-22T22:30:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:57.538 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pool=mirror 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror status mirror cluster2 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '"leader": true' 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:30:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:57.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:30:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:30:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:30:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:30:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:30:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:30:57.548 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror cluster2 2026-02-22T22:30:57.612 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "leader": true, 2026-02-22T22:30:57.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ LEADER=3 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_replay test1 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for image in "$@" 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_started cluster1:3 mirror test1 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_image_replay_state cluster1:3 mirror test1 started 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:57.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:57.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state=started 2026-02-22T22:30:57.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s 2026-02-22T22:30:57.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:57.615 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:30:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_image_replay_state cluster1:3 mirror test1 started 2026-02-22T22:30:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:30:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local test_state=started 2026-02-22T22:30:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status_result 2026-02-22T22:30:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local current_state=stopped 2026-02-22T22:30:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemons cluster1:3 rbd mirror status mirror/test1 2026-02-22T22:30:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -i state 2026-02-22T22:30:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_instance=cluster1:3 2026-02-22T22:30:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:30:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance=3 2026-02-22T22:30:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local loop_instance 2026-02-22T22:30:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 0 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 '!=' cluster1:3 ']' 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test1 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:3 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:30:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:3 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=3 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=3 2026-02-22T22:30:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:30:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test1 2026-02-22T22:30:58.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T22:30:58.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:30:58.681 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:admin_socket: invalid command 2026-02-22T22:30:58.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:30:58.682 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sleep 1 2026-02-22T22:30:59.685 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 '!=' cluster1:3 ']' 2026-02-22T22:30:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test1 2026-02-22T22:30:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:3 2026-02-22T22:30:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ shift 2026-02-22T22:30:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:30:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:30:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:30:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:30:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1 2026-02-22T22:30:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:3 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ local_cluster=cluster1 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=3 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=3 2026-02-22T22:30:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:30:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:30:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test1 2026-02-22T22:30:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ return 0 2026-02-22T22:30:59.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:30:59.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:30:59.771 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep -i Replaying 2026-02-22T22:30:59.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: "state": "Replaying" 2026-02-22T22:30:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ current_state=started 2026-02-22T22:30:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test started = started 2026-02-22T22:30:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:30:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T22:30:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local count=100 2026-02-22T22:30:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local size= 2026-02-22T22:30:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:30:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ size=4096 2026-02-22T22:30:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:30:59.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:30:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:30:59.837 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:elapsed: 0 ops: 100 ops/sec: 6250.11 bytes/sec: 24 MiB/s 2026-02-22T22:30:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_replay_complete cluster1:3 cluster2 mirror test1 2026-02-22T22:30:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:3 2026-02-22T22:30:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' journal = journal ']' 2026-02-22T22:30:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_journal_replay_complete cluster1:3 cluster2 mirror test1 2026-02-22T22:30:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local local_cluster=cluster1:3 2026-02-22T22:30:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:30:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:30:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:30:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:30:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:30:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:30:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:30:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:3 mirror test1 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:3 rbd mirror flush mirror/test1 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:3 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 '!=' cluster1:3 ']' 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test1 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:31:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:00.051 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:31:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:00.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test1 2026-02-22T22:31:00.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:31:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:31:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:31:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:31:00.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:31:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:31:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:31:00.135 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:31:00.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:31:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:31:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:31:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=21, entry_tid=101], [object_number=0, tag_tid=21, entry_tid=100], [object_number=3, tag_tid=21, entry_tid=99], [object_number=2, tag_tid=21, entry_tid=98]]], state=connected] 2026-02-22T22:31:00.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:31:00.166 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=21, entry_tid=101' 2026-02-22T22:31:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:31:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:31:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:00.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:00.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:31:00.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:31:00.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:31:00.170 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:31:00.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:31:00.171 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:31:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:31:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:31:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=21, entry_tid=101], [object_number=0, tag_tid=21, entry_tid=100], [object_number=3, tag_tid=21, entry_tid=99], [object_number=2, tag_tid=21, entry_tid=98]]], state=connected] 2026-02-22T22:31:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:31:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:00.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=21, entry_tid=101' -a 'object_number=1, tag_tid=21, entry_tid=101' = 'object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:00.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=20, entry_tid=101' '!=' '' 2026-02-22T22:31:00.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:31:00.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=20, entry_tid=101' = '' 2026-02-22T22:31:00.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_mirror_pos='object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:00.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=21, entry_tid=101' 2026-02-22T22:31:00.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:31:00.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:31:00.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_tag=21 2026-02-22T22:31:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:31:00.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:31:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_tag=20 2026-02-22T22:31:00.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=21, entry_tid=101' 2026-02-22T22:31:00.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:31:00.218 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:31:00.219 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_entry=101 2026-02-22T22:31:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo 'object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:31:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cut -d= -f 2 2026-02-22T22:31:00.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_entry=101 2026-02-22T22:31:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 21 = 20 -a 101 -le 101 2026-02-22T22:31:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ true 2026-02-22T22:31:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:31:00.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.2 2026-02-22T22:31:00.422 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:3 mirror test1 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:3 rbd mirror flush mirror/test1 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:3 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:31:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 '!=' cluster1:3 ']' 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test1 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:31:00.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:00.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:00.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:00.436 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test1 2026-02-22T22:31:00.504 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:31:00.505 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:31:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:31:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:31:00.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:31:00.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:31:00.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=21, entry_tid=101], [object_number=0, tag_tid=21, entry_tid=100], [object_number=3, tag_tid=21, entry_tid=99], [object_number=2, tag_tid=21, entry_tid=98]]], state=connected] 2026-02-22T22:31:00.535 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:31:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=21, entry_tid=101' 2026-02-22T22:31:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:31:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:31:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:31:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:31:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:31:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:31:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=21, entry_tid=101], [object_number=0, tag_tid=21, entry_tid=100], [object_number=3, tag_tid=21, entry_tid=99], [object_number=2, tag_tid=21, entry_tid=98]]], state=connected] 2026-02-22T22:31:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:31:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=21, entry_tid=101' -a 'object_number=1, tag_tid=21, entry_tid=101' = 'object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=20, entry_tid=101' '!=' 'object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:00.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:31:00.574 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.4 2026-02-22T22:31:00.977 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:3 mirror test1 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:3 rbd mirror flush mirror/test1 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:3 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:31:00.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 '!=' cluster1:3 ']' 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test1 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:31:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:31:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:31:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:31:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:31:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:31:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:31:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:31:00.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:00.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test1 2026-02-22T22:31:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:31:01.057 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:31:01.058 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:31:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:31:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:31:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:31:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:31:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=21, entry_tid=101], [object_number=0, tag_tid=21, entry_tid=100], [object_number=3, tag_tid=21, entry_tid=99], [object_number=2, tag_tid=21, entry_tid=98]]], state=connected] 2026-02-22T22:31:01.088 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:31:01.091 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=21, entry_tid=101' 2026-02-22T22:31:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:31:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:31:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:31:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:31:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:31:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:31:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:31:01.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:01.128 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:31:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:31:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:31:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=21, entry_tid=101], [object_number=0, tag_tid=21, entry_tid=100], [object_number=3, tag_tid=21, entry_tid=99], [object_number=2, tag_tid=21, entry_tid=98]]], state=connected] 2026-02-22T22:31:01.129 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=20, entry_tid=101], [object_number=0, tag_tid=20, entry_tid=100], [object_number=3, tag_tid=20, entry_tid=99], [object_number=2, tag_tid=20, entry_tid=98]]], state=connected] 2026-02-22T22:31:01.132 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=21, entry_tid=101' -a 'object_number=1, tag_tid=21, entry_tid=101' = 'object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test 'object_number=1, tag_tid=20, entry_tid=101' '!=' 'object_number=1, tag_tid=20, entry_tid=101' 2026-02-22T22:31:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:31:01.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0.8 2026-02-22T22:31:01.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ flush cluster1:3 mirror test1 2026-02-22T22:31:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:31:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:31:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n test1 ']' 2026-02-22T22:31:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T22:31:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemons cluster1:3 rbd mirror flush mirror/test1 2026-02-22T22:31:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_instance=cluster1:3 2026-02-22T22:31:01.939 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance=3 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loop_instance 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 '!=' cluster1:3 ']' 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test1 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ shift 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance 2026-02-22T22:31:01.940 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local val=cluster1:3 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster_var_name=cluster 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local instance_var_name=instance 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cluster=cluster1 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ instance=3 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval cluster=cluster1 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ eval instance=3 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:31:01.941 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:01.942 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:31:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test1 2026-02-22T22:31:02.022 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:31:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T22:31:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T22:31:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local id_regexp= 2026-02-22T22:31:02.024 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:31:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:31:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:31:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:02.025 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:31:02.026 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:31:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:31:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:31:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=21, entry_tid=101], [object_number=0, tag_tid=21, entry_tid=100], [object_number=3, tag_tid=21, entry_tid=99], [object_number=2, tag_tid=21, entry_tid=98]]], state=connected] 2026-02-22T22:31:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=21, entry_tid=101], [object_number=0, tag_tid=21, entry_tid=100], [object_number=3, tag_tid=21, entry_tid=99], [object_number=2, tag_tid=21, entry_tid=98]]], state=connected] 2026-02-22T22:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ master_pos='object_number=1, tag_tid=21, entry_tid=101' 2026-02-22T22:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T22:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T22:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local pool=mirror 2026-02-22T22:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local image=test1 2026-02-22T22:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local 'id_regexp=..*' 2026-02-22T22:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T22:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T22:31:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:31:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local status_log=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T22:31:02.071 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ tee /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:minimum_set: 0 2026-02-22T22:31:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:active_set: 0 2026-02-22T22:31:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:registered clients: 2026-02-22T22:31:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=21, entry_tid=101], [object_number=0, tag_tid=21, entry_tid=100], [object_number=3, tag_tid=21, entry_tid=99], [object_number=2, tag_tid=21, entry_tid=98]]], state=connected] 2026-02-22T22:31:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: [id=3f048100-f069-4ca5-a089-674a43491519, commit_position=[positions=[[object_number=1, tag_tid=21, entry_tid=101], [object_number=0, tag_tid=21, entry_tid=100], [object_number=3, tag_tid=21, entry_tid=99], [object_number=2, tag_tid=21, entry_tid=98]]], state=connected] 2026-02-22T22:31:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.status 2026-02-22T22:31:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ mirror_pos='object_number=1, tag_tid=21, entry_tid=101' 2026-02-22T22:31:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n 'object_number=1, tag_tid=21, entry_tid=101' -a 'object_number=1, tag_tid=21, entry_tid=101' = 'object_number=1, tag_tid=21, entry_tid=101' 2026-02-22T22:31:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:31:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ hostname -s 2026-02-22T22:31:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.3 on trial190' 2026-02-22T22:31:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:31:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:31:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:31:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.3 on trial190' 2026-02-22T22:31:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:31:02.105 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:31:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 'mirror.3 on trial190' 2026-02-22T22:31:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:31:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:31:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern=primary_position 2026-02-22T22:31:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'service_pattern=mirror.3 on trial190' 2026-02-22T22:31:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:31:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:31:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T22:31:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:31:03.143 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:31:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:31:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":20},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":20}} 2026-02-22T22:31:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: service: mirror.3 on trial190 2026-02-22T22:31:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:59' 2026-02-22T22:31:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:31:03.144 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: up+replaying 2026-02-22T22:31:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:31:03.145 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":20},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":20}} 2026-02-22T22:31:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 'mirror.3 on trial190' ']' 2026-02-22T22:31:03.146 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: *mirror.3 on trial190' 2026-02-22T22:31:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: service: mirror.3 on trial190 2026-02-22T22:31:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:31:03.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:31:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:59' 2026-02-22T22:31:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 22:30:59' 2026-02-22T22:31:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:31:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=up+replaying 2026-02-22T22:31:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local 'prev_last_update=2026-02-22 22:30:59' 2026-02-22T22:31:03.150 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:31:03.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:31:03.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:31:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='WARNINGOKWARNING1245604567mirror.3trial19019.2.3-689-gf90f4eaetrueOKtest1ccef1051-f64a-4f10-99c3-dcdd8d739edfup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":20},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":20}}45604567mirror.3trial1902026-02-22 22:30:59test108c346bd9-9262-4edd-8b07-709d0b842405up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45604567mirror.3trial1902026-02-22 22:30:59test11538c249f-3fcb-4b38-9230-5300b8fd0ac1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45604567mirror.3trial1902026-02-22 22:30:59test135016d2e9-08b5-49f5-b76a-6e2480467b80up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45604567mirror.3trial1902026-02-22 22:30:59test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}45604567mirror.3trial1902026-02-22 22:30:59test35b142ce3-e021-4535-96a0-b1d925ba9b83up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45604567mirror.3trial1902026-02-22 22:30:59test41335d58d-7238-49a4-980e-749ad53dd763up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}45604567mirror.3trial1902026-02-22 22:30:59test527d435c7-b11b-4cdb-9cab-ccb4d4e8032eup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45604567mirror.3trial1902026-02-22 22:30:59test69f2c857c-e2fe-4ac6-9969-3ab6676435e7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45604567mirror.3trial1902026-02-22 22:30:59test726707d20-3b5a-4c06-983c-c59b35d21959up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45604567mirror.3trial1902026-02-22 22:30:59test8cf8c8374-a30a-4b73-92b3-fc0829311a96up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45604567mirror.3trial1902026-02-22 22:30:59test94728bd8a-2d02-4dec-90a0-478e08066247up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}45604567mirror.3trial1902026-02-22 22:30:59' 2026-02-22T22:31:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:31:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:31:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update='2026-02-22 22:30:59' 2026-02-22T22:31:03.217 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:31:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=up+replaying 2026-02-22T22:31:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo up+replaying 2026-02-22T22:31:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep up+replaying 2026-02-22T22:31:03.222 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:up+replaying 2026-02-22T22:31:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:31:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:31:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:31:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T22:31:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:31:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:31:04.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T22:31:04.224 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:31:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:31:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local description_pattern= 2026-02-22T22:31:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local service_pattern= 2026-02-22T22:31:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:31:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:31:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T22:31:04.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='test1: 2026-02-22T22:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: global_id: ccef1051-f64a-4f10-99c3-dcdd8d739edf 2026-02-22T22:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: down+unknown 2026-02-22T22:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: status not found 2026-02-22T22:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22T22:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: peer_sites: 2026-02-22T22:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: name: cluster1 2026-02-22T22:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: state: up+replaying 2026-02-22T22:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":20},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":20}} 2026-02-22T22:31:04.257 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: last_update: 2026-02-22 22:30:59' 2026-02-22T22:31:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T22:31:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: state: down+unknown 2026-02-22T22:31:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^ description: .*' 2026-02-22T22:31:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout: description: status not found 2026-02-22T22:31:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:31:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep '^up+' 2026-02-22T22:31:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep 'service: ' 2026-02-22T22:31:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update 2026-02-22T22:31:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:31:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:31:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T22:31:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:31:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local state_pattern=down+unknown 2026-02-22T22:31:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local prev_last_update= 2026-02-22T22:31:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local status 2026-02-22T22:31:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ CEPH_ARGS= 2026-02-22T22:31:04.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:31:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ status='UNKNOWNUNKNOWNOK12test1ccef1051-f64a-4f10-99c3-dcdd8d739edfdown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":20},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":20}}2026-02-22 22:30:59test108c346bd9-9262-4edd-8b07-709d0b842405down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:59test11538c249f-3fcb-4b38-9230-5300b8fd0ac1down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:59test135016d2e9-08b5-49f5-b76a-6e2480467b80down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:59test2b9ac5e9e-d9c1-47bc-91b9-1d5ef05311f0down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":6}}2026-02-22 22:30:59test35b142ce3-e021-4535-96a0-b1d925ba9b83down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:59test41335d58d-7238-49a4-980e-749ad53dd763down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":3}}2026-02-22 22:30:59test527d435c7-b11b-4cdb-9cab-ccb4d4e8032edown+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:59test69f2c857c-e2fe-4ac6-9969-3ab6676435e7down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:59test726707d20-3b5a-4c06-983c-c59b35d21959down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:59test8cf8c8374-a30a-4b73-92b3-fc0829311a96down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:59test94728bd8a-2d02-4dec-90a0-478e08066247down+unknownstatus not foundcluster13f048100-f069-4ca5-a089-674a43491519up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":2}}2026-02-22 22:30:59' 2026-02-22T22:31:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local last_update state 2026-02-22T22:31:04.318 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T22:31:04.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ last_update= 2026-02-22T22:31:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T22:31:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ state=down+unknown 2026-02-22T22:31:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo down+unknown 2026-02-22T22:31:04.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ grep down+unknown 2026-02-22T22:31:04.325 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:down+unknown 2026-02-22T22:31:04.326 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:31:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:31:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ compare_images mirror test1 2026-02-22T22:31:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pool=mirror 2026-02-22T22:31:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local image=test1 2026-02-22T22:31:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local ret=0 2026-02-22T22:31:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T22:31:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T22:31:04.327 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:31:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local rmt_export=/tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:31:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T22:31:04.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T22:31:04.329 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:31:04.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local loc_export=/tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:31:04.330 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:31:04.331 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export 2026-02-22T22:31:04.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:31:04.463 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:31:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:31:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cmp /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:31:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /tmp/tmp.ek6CoRRxsq/cluster2-mirror-test1.export /tmp/tmp.ek6CoRRxsq/cluster1-mirror-test1.export 2026-02-22T22:31:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ return 0 2026-02-22T22:31:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:3 2026-02-22T22:31:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:31:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:3 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:3 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=3 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=3 2026-02-22T22:31:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:31:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:31:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:31:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid=36406 2026-02-22T22:31:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n 36406 ']' 2026-02-22T22:31:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ kill 36406 2026-02-22T22:31:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:31:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 0 2026-02-22T22:31:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:31:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36406 '$2 == pid {print; exit 1}' 2026-02-22T22:31:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:ubuntu 36406 0.4 0.0 780032 57412 ? Ssl 22:29 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:31:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:31:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ sleep 1 2026-02-22T22:31:05.750 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:31:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36406 '$2 == pid {print; exit 1}' 2026-02-22T22:31:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ break 2026-02-22T22:31:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ps auxww 2026-02-22T22:31:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ awk -v pid=36406 '$2 == pid {print; exit 1}' 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:31:05.793 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:05.794 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T22:31:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:31:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:31:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:31:05.796 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:31:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:05.808 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:3 2026-02-22T22:31:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:3 2026-02-22T22:31:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:05.809 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:05.810 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:31:05.820 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:31:05.821 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:31:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ cleanup 0 2026-02-22T22:31:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local error_code=0 2026-02-22T22:31:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ set +e 2026-02-22T22:31:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 -ne 0 ']' 2026-02-22T22:31:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z '' ']' 2026-02-22T22:31:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster instance 2026-02-22T22:31:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:31:05.822 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T22:31:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:pool 'mirror' does not exist 2026-02-22T22:31:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:31:06.178 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T22:31:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:pool 'mirror' does not exist 2026-02-22T22:31:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:31:06.719 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T22:31:07.181 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:pool 'mirror_parent' does not exist 2026-02-22T22:31:07.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ CEPH_ARGS= 2026-02-22T22:31:07.193 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T22:31:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:pool 'mirror_parent' does not exist 2026-02-22T22:31:07.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T22:31:07.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirrors cluster1 2026-02-22T22:31:07.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1 2026-02-22T22:31:07.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:07.713 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:31:07.714 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:07.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T22:31:07.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:0 2026-02-22T22:31:07.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:07.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:07.715 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:31:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:0 2026-02-22T22:31:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:31:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:0 2026-02-22T22:31:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:07.717 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:31:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:31:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:31:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:31:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:31:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:31:07.718 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:31:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:31:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:07.730 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:07.732 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:31:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:31:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:31:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:31:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:31:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:31:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:31:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:31:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:31:07.744 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:31:07.745 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:31:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:31:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:31:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:31:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:31:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:07.747 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:31:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:31:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:31:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:31:07.748 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:31:07.758 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:31:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:31:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:31:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:0 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:0 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:31:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:31:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:31:07.772 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:31:07.773 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:31:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T22:31:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:1 2026-02-22T22:31:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:07.774 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T22:31:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:1 2026-02-22T22:31:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:31:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:1 2026-02-22T22:31:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:31:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:31:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:31:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:31:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:31:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:31:07.777 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:31:07.785 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:31:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:31:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:31:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:31:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:31:07.790 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:31:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:31:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:31:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T22:31:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:31:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:31:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:31:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:31:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:31:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:31:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:31:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:31:07.813 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:31:07.814 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:31:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:1 2026-02-22T22:31:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:1 2026-02-22T22:31:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:1 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:31:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:31:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:31:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:31:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T22:31:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:2 2026-02-22T22:31:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T22:31:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:2 2026-02-22T22:31:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:31:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:2 2026-02-22T22:31:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:31:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:31:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:31:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:31:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:31:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:31:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:31:07.840 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:31:07.841 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:31:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:31:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:31:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:31:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:31:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:31:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:31:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T22:31:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:31:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:31:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:31:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:31:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:31:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:31:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:31:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:31:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:31:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:31:07.870 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:2 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:2 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:2 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:31:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:31:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:31:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:31:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:31:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T22:31:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:3 2026-02-22T22:31:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T22:31:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:3 2026-02-22T22:31:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:3 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=3 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=3 2026-02-22T22:31:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:31:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:31:07.894 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:31:07.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:07.895 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:07.896 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:07.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:31:07.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:31:07.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:07.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:31:07.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:31:07.897 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:07.898 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:31:07.908 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:07.909 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T22:31:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:31:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:07.910 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:31:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:07.921 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:31:07.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:3 2026-02-22T22:31:07.923 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:3 2026-02-22T22:31:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:31:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:3 2026-02-22T22:31:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:31:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:31:07.924 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:07.925 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:31:07.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:31:07.934 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:31:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:4 '' 2026-02-22T22:31:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:4 2026-02-22T22:31:07.935 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:4 2026-02-22T22:31:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:4 2026-02-22T22:31:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:31:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:4 2026-02-22T22:31:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:31:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 4 = cluster1:4 ']' 2026-02-22T22:31:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:31:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:31:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=4 2026-02-22T22:31:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=4 2026-02-22T22:31:07.938 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.4 'pid file' 2026-02-22T22:31:07.948 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.pid 2026-02-22T22:31:07.948 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.pid 2026-02-22T22:31:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster1 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:31:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:31:07.952 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:31:07.961 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:31:07.962 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:31:07.963 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:4 cluster2 2026-02-22T22:31:07.963 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:4 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 local_cluster instance 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:07.964 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:31:07.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:31:07.965 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'admin socket' 2026-02-22T22:31:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:31:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.asok 2026-02-22T22:31:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:4 2026-02-22T22:31:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:4 2026-02-22T22:31:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:4 cluster instance 2026-02-22T22:31:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:4 2026-02-22T22:31:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:31:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster1:4 ']' 2026-02-22T22:31:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:31:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:31:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:31:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.4 'pid file' 2026-02-22T22:31:07.989 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.pid 2026-02-22T22:31:07.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.4.pid 2026-02-22T22:31:07.990 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:07.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:5 '' 2026-02-22T22:31:07.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:5 2026-02-22T22:31:07.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:07.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:07.991 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:5 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:5 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:5 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 5 = cluster1:5 ']' 2026-02-22T22:31:07.993 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:31:07.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:31:07.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=5 2026-02-22T22:31:07.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=5 2026-02-22T22:31:07.994 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.5 'pid file' 2026-02-22T22:31:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.pid 2026-02-22T22:31:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.pid 2026-02-22T22:31:08.004 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:08.004 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:08.004 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster1 2026-02-22T22:31:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:31:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:31:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:31:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:31:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:31:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:31:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:31:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:31:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:31:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:31:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:5 cluster2 2026-02-22T22:31:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:5 2026-02-22T22:31:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 local_cluster instance 2026-02-22T22:31:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:31:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:31:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:31:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:31:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:31:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'admin socket' 2026-02-22T22:31:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:31:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.asok 2026-02-22T22:31:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:5 2026-02-22T22:31:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:5 2026-02-22T22:31:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:5 cluster instance 2026-02-22T22:31:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:5 2026-02-22T22:31:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:31:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster1:5 ']' 2026-02-22T22:31:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:31:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:31:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:31:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.5 'pid file' 2026-02-22T22:31:08.040 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.pid 2026-02-22T22:31:08.041 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.5.pid 2026-02-22T22:31:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster1:6 '' 2026-02-22T22:31:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster1:6 2026-02-22T22:31:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster1:6 2026-02-22T22:31:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster1:6 2026-02-22T22:31:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:31:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster1:6 2026-02-22T22:31:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:31:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 6 = cluster1:6 ']' 2026-02-22T22:31:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster1 2026-02-22T22:31:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster1 2026-02-22T22:31:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=6 2026-02-22T22:31:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=6 2026-02-22T22:31:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.6 'pid file' 2026-02-22T22:31:08.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.pid 2026-02-22T22:31:08.053 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.pid 2026-02-22T22:31:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster1 2026-02-22T22:31:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:31:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:08.055 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:31:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:31:08.065 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:31:08.066 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:31:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster1:6 cluster2 2026-02-22T22:31:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster1:6 2026-02-22T22:31:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 local_cluster instance 2026-02-22T22:31:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:31:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:31:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:31:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster1 2026-02-22T22:31:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster1 2026-02-22T22:31:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:31:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:31:08.069 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'admin socket' 2026-02-22T22:31:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:31:08.080 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.asok 2026-02-22T22:31:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster1:6 2026-02-22T22:31:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1:6 2026-02-22T22:31:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster1:6 cluster instance 2026-02-22T22:31:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster1:6 2026-02-22T22:31:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster1 2026-02-22T22:31:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:31:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster1:6 ']' 2026-02-22T22:31:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster1 2026-02-22T22:31:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster1 2026-02-22T22:31:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:31:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:31:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.6 'pid file' 2026-02-22T22:31:08.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.pid 2026-02-22T22:31:08.093 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.6.pid 2026-02-22T22:31:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T22:31:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirrors cluster2 2026-02-22T22:31:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2 2026-02-22T22:31:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:08.095 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ seq 0 6 2026-02-22T22:31:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster2:0 '' 2026-02-22T22:31:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2:0 2026-02-22T22:31:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster2:0 2026-02-22T22:31:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster2:0 2026-02-22T22:31:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:31:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster2:0 2026-02-22T22:31:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:08.098 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:31:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:31:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster2 2026-02-22T22:31:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster2 2026-02-22T22:31:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=0 2026-02-22T22:31:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=0 2026-02-22T22:31:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T22:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:31:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-22T22:31:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:31:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:0 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:31:08.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:31:08.121 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:0 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:31:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:31:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:31:08.124 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:31:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:31:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster2:0 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2:0 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:0 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=0 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:31:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster2 2026-02-22T22:31:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=0 2026-02-22T22:31:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=0 2026-02-22T22:31:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T22:31:08.147 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:31:08.148 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:31:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster2:1 '' 2026-02-22T22:31:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2:1 2026-02-22T22:31:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster2:1 2026-02-22T22:31:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster2:1 2026-02-22T22:31:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:31:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster2:1 2026-02-22T22:31:08.151 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T22:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster2 2026-02-22T22:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster2 2026-02-22T22:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=1 2026-02-22T22:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=1 2026-02-22T22:31:08.152 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T22:31:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:31:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:31:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-02-22T22:31:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:31:08.162 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:1 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:31:08.163 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:31:08.164 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:31:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:31:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:31:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:31:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:31:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:31:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:1 2026-02-22T22:31:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.176 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:31:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:31:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:31:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:31:08.177 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:31:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:31:08.186 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:31:08.187 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster2:1 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2:1 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:1 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=1 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster2 2026-02-22T22:31:08.188 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=1 2026-02-22T22:31:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=1 2026-02-22T22:31:08.189 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T22:31:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:31:08.200 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:31:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster2:2 '' 2026-02-22T22:31:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2:2 2026-02-22T22:31:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster2:2 2026-02-22T22:31:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster2:2 2026-02-22T22:31:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:31:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster2:2 2026-02-22T22:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T22:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster2 2026-02-22T22:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster2 2026-02-22T22:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=2 2026-02-22T22:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=2 2026-02-22T22:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T22:31:08.212 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:31:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:31:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-02-22T22:31:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:31:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:31:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:2 2026-02-22T22:31:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:31:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:31:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:31:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:31:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:31:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:31:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:31:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:2 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:31:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:31:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:31:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:31:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:31:08.240 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:31:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster2:2 2026-02-22T22:31:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2:2 2026-02-22T22:31:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:31:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:2 2026-02-22T22:31:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.242 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=2 2026-02-22T22:31:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:31:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster2 2026-02-22T22:31:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=2 2026-02-22T22:31:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=2 2026-02-22T22:31:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T22:31:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:31:08.253 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:31:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster2:3 '' 2026-02-22T22:31:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2:3 2026-02-22T22:31:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster2:3 2026-02-22T22:31:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster2:3 2026-02-22T22:31:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:08.255 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster2:3 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster2 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster2 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=3 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=3 2026-02-22T22:31:08.256 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T22:31:08.264 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:31:08.265 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:31:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-02-22T22:31:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:31:08.267 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:3 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:31:08.277 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:31:08.278 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:31:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:31:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:31:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:31:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:3 2026-02-22T22:31:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:31:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:31:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:31:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:31:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster2:3 2026-02-22T22:31:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2:3 2026-02-22T22:31:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:31:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:3 2026-02-22T22:31:08.294 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=3 2026-02-22T22:31:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:31:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster2 2026-02-22T22:31:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=3 2026-02-22T22:31:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=3 2026-02-22T22:31:08.295 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T22:31:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:31:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:31:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster2:4 '' 2026-02-22T22:31:08.306 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2:4 2026-02-22T22:31:08.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:08.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:08.307 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster2:4 2026-02-22T22:31:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster2:4 2026-02-22T22:31:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:08.308 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster2:4 cluster instance 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster2:4 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 4 = cluster2:4 ']' 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster2 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster2 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=4 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=4 2026-02-22T22:31:08.309 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.4 'pid file' 2026-02-22T22:31:08.319 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.4.pid 2026-02-22T22:31:08.320 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.4.pid 2026-02-22T22:31:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:4 cluster1 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:4 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:4 local_cluster instance 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:4 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster2:4 ']' 2026-02-22T22:31:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:31:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:31:08.324 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.4 'admin socket' 2026-02-22T22:31:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.4.asok 2026-02-22T22:31:08.333 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.4.asok 2026-02-22T22:31:08.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:4 cluster2 2026-02-22T22:31:08.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:4 2026-02-22T22:31:08.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:08.335 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:4 local_cluster instance 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:4 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster2:4 ']' 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:31:08.336 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.4 'admin socket' 2026-02-22T22:31:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.4.asok 2026-02-22T22:31:08.347 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.4.asok 2026-02-22T22:31:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster2:4 2026-02-22T22:31:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2:4 2026-02-22T22:31:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:4 cluster instance 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:4 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=4 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 4 = cluster2:4 ']' 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster2 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=4 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=4 2026-02-22T22:31:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.4 'pid file' 2026-02-22T22:31:08.359 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.4.pid 2026-02-22T22:31:08.360 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.4.pid 2026-02-22T22:31:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster2:5 '' 2026-02-22T22:31:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2:5 2026-02-22T22:31:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:08.361 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster2:5 2026-02-22T22:31:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster2:5 2026-02-22T22:31:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster2:5 cluster instance 2026-02-22T22:31:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster2:5 2026-02-22T22:31:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:31:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 5 = cluster2:5 ']' 2026-02-22T22:31:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster2 2026-02-22T22:31:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster2 2026-02-22T22:31:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=5 2026-02-22T22:31:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=5 2026-02-22T22:31:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.5 'pid file' 2026-02-22T22:31:08.372 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.5.pid 2026-02-22T22:31:08.373 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.5.pid 2026-02-22T22:31:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:08.375 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:08.376 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:5 cluster1 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:5 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:5 local_cluster instance 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:5 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster2:5 ']' 2026-02-22T22:31:08.377 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:31:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:31:08.378 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.5 'admin socket' 2026-02-22T22:31:08.387 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.5.asok 2026-02-22T22:31:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.5.asok 2026-02-22T22:31:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:5 cluster2 2026-02-22T22:31:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:5 2026-02-22T22:31:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:5 local_cluster instance 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:5 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster2:5 ']' 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:31:08.390 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.5 'admin socket' 2026-02-22T22:31:08.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.5.asok 2026-02-22T22:31:08.401 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.5.asok 2026-02-22T22:31:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster2:5 2026-02-22T22:31:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2:5 2026-02-22T22:31:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:5 cluster instance 2026-02-22T22:31:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:5 2026-02-22T22:31:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=5 2026-02-22T22:31:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 5 = cluster2:5 ']' 2026-02-22T22:31:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster2 2026-02-22T22:31:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=5 2026-02-22T22:31:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=5 2026-02-22T22:31:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.5 'pid file' 2026-02-22T22:31:08.415 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.5.pid 2026-02-22T22:31:08.416 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.5.pid 2026-02-22T22:31:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:31:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ stop_mirror cluster2:6 '' 2026-02-22T22:31:08.417 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local cluster=cluster2:6 2026-02-22T22:31:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local sig= 2026-02-22T22:31:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test -n '' 2026-02-22T22:31:08.418 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ local pid 2026-02-22T22:31:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ daemon_pid_file cluster2:6 2026-02-22T22:31:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster=cluster2:6 2026-02-22T22:31:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance 2026-02-22T22:31:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ set_cluster_instance cluster2:6 cluster instance 2026-02-22T22:31:08.419 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local val=cluster2:6 2026-02-22T22:31:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:31:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local instance_var_name=instance 2026-02-22T22:31:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:31:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ '[' 6 = cluster2:6 ']' 2026-02-22T22:31:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval cluster=cluster2 2026-02-22T22:31:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ cluster=cluster2 2026-02-22T22:31:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ eval instance=6 2026-02-22T22:31:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ instance=6 2026-02-22T22:31:08.420 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.6 'pid file' 2026-02-22T22:31:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.6.pid 2026-02-22T22:31:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.6.pid 2026-02-22T22:31:08.431 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ pid= 2026-02-22T22:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ : 2026-02-22T22:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -n '' ']' 2026-02-22T22:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:6 cluster1 2026-02-22T22:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:6 2026-02-22T22:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster1 2026-02-22T22:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:6 local_cluster instance 2026-02-22T22:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:6 2026-02-22T22:31:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster2:6 ']' 2026-02-22T22:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:31:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.6 'admin socket' 2026-02-22T22:31:08.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.6.asok 2026-02-22T22:31:08.444 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.6.asok 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_asok_file cluster2:6 cluster2 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local local_cluster=cluster2:6 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:6 local_cluster instance 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:6 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:31:08.446 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster2:6 ']' 2026-02-22T22:31:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval local_cluster=cluster2 2026-02-22T22:31:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ local_cluster=cluster2 2026-02-22T22:31:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:31:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:31:08.447 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.6 'admin socket' 2026-02-22T22:31:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.6.asok 2026-02-22T22:31:08.457 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.6.asok 2026-02-22T22:31:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ daemon_pid_file cluster2:6 2026-02-22T22:31:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster=cluster2:6 2026-02-22T22:31:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance 2026-02-22T22:31:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ set_cluster_instance cluster2:6 cluster instance 2026-02-22T22:31:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local val=cluster2:6 2026-02-22T22:31:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local cluster_var_name=cluster 2026-02-22T22:31:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ local instance_var_name=instance 2026-02-22T22:31:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ cluster=cluster2 2026-02-22T22:31:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ instance=6 2026-02-22T22:31:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ '[' 6 = cluster2:6 ']' 2026-02-22T22:31:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval cluster=cluster2 2026-02-22T22:31:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ cluster=cluster2 2026-02-22T22:31:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ eval instance=6 2026-02-22T22:31:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ instance=6 2026-02-22T22:31:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.6 'pid file' 2026-02-22T22:31:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.6.pid 2026-02-22T22:31:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.6.pid 2026-02-22T22:31:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' -z 1 ']' 2026-02-22T22:31:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ test '' = /tmp/tmp.ek6CoRRxsq 2026-02-22T22:31:08.472 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ rm -Rf /tmp/tmp.ek6CoRRxsq 2026-02-22T22:31:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:31:08.473 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ echo OK 2026-02-22T22:31:08.474 INFO:tasks.workunit.cluster1.client.mirror.trial190.stdout:OK 2026-02-22T22:31:08.475 INFO:tasks.workunit.cluster1.client.mirror.trial190.stderr:+ exit 0 2026-02-22T22:31:08.475 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-22T22:31:08.475 DEBUG:teuthology.orchestra.run.trial190:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-22T22:31:08.523 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_ha.sh'] on cluster1.client.mirror... 2026-02-22T22:31:08.524 DEBUG:teuthology.orchestra.run.trial190:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-22T22:31:08.872 DEBUG:teuthology.parallel:result is None 2026-02-22T22:31:08.872 DEBUG:teuthology.orchestra.run.trial190:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:31:08.881 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:31:08.881 DEBUG:teuthology.orchestra.run.trial190:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:31:08.927 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:31:08.927 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T22:31:09.002 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T22:31:09.003 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:31:09.150 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:31:09.151 INFO:teuthology.orchestra.run.trial190.stderr:dumped all 2026-02-22T22:31:09.161 INFO:teuthology.orchestra.run.trial190.stdout:{"pg_ready":true,"pg_map":{"version":260,"stamp":"2026-02-22T22:31:08.333705+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":234,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":140600,"kb_used_data":26432,"kb_used_omap":4,"kb_used_meta":114107,"kb_avail":282974920,"statfs":{"total":289910292480,"available":289766318080,"internally_reserved":0,"allocated":27066368,"data_stored":26134060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":116845906},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1230467670,"num_objects":-476,"num_object_clones":0,"num_object_copies":-952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-476,"num_whiteouts":0,"num_read":-42682,"num_read_kb":-343005,"num_write":-5609,"num_write_kb":-30651,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-35,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001291"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":110,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.938303+0000","last_change":"2026-02-22T22:24:36.318108+0000","last_active":"2026-02-22T22:29:58.938303+0000","last_peered":"2026-02-22T22:29:58.938303+0000","last_clean":"2026-02-22T22:29:58.938303+0000","last_became_active":"2026-02-22T22:24:34.316382+0000","last_became_peered":"2026-02-22T22:24:34.316382+0000","last_unstale":"2026-02-22T22:29:58.938303+0000","last_undegraded":"2026-02-22T22:29:58.938303+0000","last_fullsized":"2026-02-22T22:29:58.938303+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:30:56.453533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057076399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":111,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.938800+0000","last_change":"2026-02-22T22:24:36.318553+0000","last_active":"2026-02-22T22:29:58.938800+0000","last_peered":"2026-02-22T22:29:58.938800+0000","last_clean":"2026-02-22T22:29:58.938800+0000","last_became_active":"2026-02-22T22:24:34.315465+0000","last_became_peered":"2026-02-22T22:24:34.315465+0000","last_unstale":"2026-02-22T22:29:58.938800+0000","last_undegraded":"2026-02-22T22:29:58.938800+0000","last_fullsized":"2026-02-22T22:29:58.938800+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:27:00.271117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00102524,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.938899+0000","last_change":"2026-02-22T22:24:36.318688+0000","last_active":"2026-02-22T22:29:58.938899+0000","last_peered":"2026-02-22T22:29:58.938899+0000","last_clean":"2026-02-22T22:29:58.938899+0000","last_became_active":"2026-02-22T22:24:34.316505+0000","last_became_peered":"2026-02-22T22:24:34.316505+0000","last_unstale":"2026-02-22T22:29:58.938899+0000","last_undegraded":"2026-02-22T22:29:58.938899+0000","last_fullsized":"2026-02-22T22:29:58.938899+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:54.581297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001096383,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.936914+0000","last_change":"2026-02-22T22:24:36.318328+0000","last_active":"2026-02-22T22:29:58.936914+0000","last_peered":"2026-02-22T22:29:58.936914+0000","last_clean":"2026-02-22T22:29:58.936914+0000","last_became_active":"2026-02-22T22:24:34.316336+0000","last_became_peered":"2026-02-22T22:24:34.316336+0000","last_unstale":"2026-02-22T22:29:58.936914+0000","last_undegraded":"2026-02-22T22:29:58.936914+0000","last_fullsized":"2026-02-22T22:29:58.936914+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:42:20.387771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079384900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":116,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.146958+0000","last_change":"2026-02-22T22:24:36.315539+0000","last_active":"2026-02-22T22:31:07.146958+0000","last_peered":"2026-02-22T22:31:07.146958+0000","last_clean":"2026-02-22T22:31:07.146958+0000","last_became_active":"2026-02-22T22:24:34.315494+0000","last_became_peered":"2026-02-22T22:24:34.315494+0000","last_unstale":"2026-02-22T22:31:07.146958+0000","last_undegraded":"2026-02-22T22:31:07.146958+0000","last_fullsized":"2026-02-22T22:31:07.146958+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:57:09.391401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278449,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":114,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:06.655510+0000","last_change":"2026-02-22T22:24:37.193154+0000","last_active":"2026-02-22T22:31:06.655510+0000","last_peered":"2026-02-22T22:31:06.655510+0000","last_clean":"2026-02-22T22:31:06.655510+0000","last_became_active":"2026-02-22T22:24:34.315699+0000","last_became_peered":"2026-02-22T22:24:34.315699+0000","last_unstale":"2026-02-22T22:31:06.655510+0000","last_undegraded":"2026-02-22T22:31:06.655510+0000","last_fullsized":"2026-02-22T22:31:06.655510+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:06:14.131443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028952900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":114,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:06.655962+0000","last_change":"2026-02-22T22:24:37.193148+0000","last_active":"2026-02-22T22:31:06.655962+0000","last_peered":"2026-02-22T22:31:06.655962+0000","last_clean":"2026-02-22T22:31:06.655962+0000","last_became_active":"2026-02-22T22:24:34.315765+0000","last_became_peered":"2026-02-22T22:24:34.315765+0000","last_unstale":"2026-02-22T22:31:06.655962+0000","last_undegraded":"2026-02-22T22:31:06.655962+0000","last_fullsized":"2026-02-22T22:31:06.655962+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:23:11.125424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278349,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":112,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.936998+0000","last_change":"2026-02-22T22:24:36.318064+0000","last_active":"2026-02-22T22:29:58.936998+0000","last_peered":"2026-02-22T22:29:58.936998+0000","last_clean":"2026-02-22T22:29:58.936998+0000","last_became_active":"2026-02-22T22:24:34.315206+0000","last_became_peered":"2026-02-22T22:24:34.315206+0000","last_unstale":"2026-02-22T22:29:58.936998+0000","last_undegraded":"2026-02-22T22:29:58.936998+0000","last_fullsized":"2026-02-22T22:29:58.936998+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:48:25.916897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056075500000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":201,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.936555+0000","last_change":"2026-02-22T22:24:30.304133+0000","last_active":"2026-02-22T22:29:58.936555+0000","last_peered":"2026-02-22T22:29:58.936555+0000","last_clean":"2026-02-22T22:29:58.936555+0000","last_became_active":"2026-02-22T22:24:30.303257+0000","last_became_peered":"2026-02-22T22:24:30.303257+0000","last_unstale":"2026-02-22T22:29:58.936555+0000","last_undegraded":"2026-02-22T22:29:58.936555+0000","last_fullsized":"2026-02-22T22:29:58.936555+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_clean_scrub_stamp":"2026-02-22T22:24:29.295633+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:57:58.047355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738451,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":39948,"kb_used_data":504,"kb_used_omap":1,"kb_used_meta":39422,"kb_avail":94331892,"statfs":{"total":96636764160,"available":96595857408,"internally_reserved":0,"allocated":516096,"data_stored":237656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40368582},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738450,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":60124,"kb_used_data":24844,"kb_used_omap":1,"kb_used_meta":35262,"kb_avail":94311716,"statfs":{"total":96636764160,"available":96575197184,"internally_reserved":0,"allocated":25440256,"data_stored":25068380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36108742},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738450,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40528,"kb_used_data":1084,"kb_used_omap":1,"kb_used_meta":39422,"kb_avail":94331312,"statfs":{"total":96636764160,"available":96595263488,"internally_reserved":0,"allocated":1110016,"data_stored":828024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40368582},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:31:09.161 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:31:09.317 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:31:09.317 INFO:teuthology.orchestra.run.trial190.stderr:dumped all 2026-02-22T22:31:09.327 INFO:teuthology.orchestra.run.trial190.stdout:{"pg_ready":true,"pg_map":{"version":260,"stamp":"2026-02-22T22:31:08.333705+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":234,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":140600,"kb_used_data":26432,"kb_used_omap":4,"kb_used_meta":114107,"kb_avail":282974920,"statfs":{"total":289910292480,"available":289766318080,"internally_reserved":0,"allocated":27066368,"data_stored":26134060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":116845906},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1230467670,"num_objects":-476,"num_object_clones":0,"num_object_copies":-952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-476,"num_whiteouts":0,"num_read":-42682,"num_read_kb":-343005,"num_write":-5609,"num_write_kb":-30651,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-35,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001291"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":110,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.938303+0000","last_change":"2026-02-22T22:24:36.318108+0000","last_active":"2026-02-22T22:29:58.938303+0000","last_peered":"2026-02-22T22:29:58.938303+0000","last_clean":"2026-02-22T22:29:58.938303+0000","last_became_active":"2026-02-22T22:24:34.316382+0000","last_became_peered":"2026-02-22T22:24:34.316382+0000","last_unstale":"2026-02-22T22:29:58.938303+0000","last_undegraded":"2026-02-22T22:29:58.938303+0000","last_fullsized":"2026-02-22T22:29:58.938303+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:30:56.453533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057076399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":111,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.938800+0000","last_change":"2026-02-22T22:24:36.318553+0000","last_active":"2026-02-22T22:29:58.938800+0000","last_peered":"2026-02-22T22:29:58.938800+0000","last_clean":"2026-02-22T22:29:58.938800+0000","last_became_active":"2026-02-22T22:24:34.315465+0000","last_became_peered":"2026-02-22T22:24:34.315465+0000","last_unstale":"2026-02-22T22:29:58.938800+0000","last_undegraded":"2026-02-22T22:29:58.938800+0000","last_fullsized":"2026-02-22T22:29:58.938800+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:27:00.271117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00102524,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.938899+0000","last_change":"2026-02-22T22:24:36.318688+0000","last_active":"2026-02-22T22:29:58.938899+0000","last_peered":"2026-02-22T22:29:58.938899+0000","last_clean":"2026-02-22T22:29:58.938899+0000","last_became_active":"2026-02-22T22:24:34.316505+0000","last_became_peered":"2026-02-22T22:24:34.316505+0000","last_unstale":"2026-02-22T22:29:58.938899+0000","last_undegraded":"2026-02-22T22:29:58.938899+0000","last_fullsized":"2026-02-22T22:29:58.938899+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:54.581297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001096383,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.936914+0000","last_change":"2026-02-22T22:24:36.318328+0000","last_active":"2026-02-22T22:29:58.936914+0000","last_peered":"2026-02-22T22:29:58.936914+0000","last_clean":"2026-02-22T22:29:58.936914+0000","last_became_active":"2026-02-22T22:24:34.316336+0000","last_became_peered":"2026-02-22T22:24:34.316336+0000","last_unstale":"2026-02-22T22:29:58.936914+0000","last_undegraded":"2026-02-22T22:29:58.936914+0000","last_fullsized":"2026-02-22T22:29:58.936914+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:42:20.387771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079384900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":116,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.146958+0000","last_change":"2026-02-22T22:24:36.315539+0000","last_active":"2026-02-22T22:31:07.146958+0000","last_peered":"2026-02-22T22:31:07.146958+0000","last_clean":"2026-02-22T22:31:07.146958+0000","last_became_active":"2026-02-22T22:24:34.315494+0000","last_became_peered":"2026-02-22T22:24:34.315494+0000","last_unstale":"2026-02-22T22:31:07.146958+0000","last_undegraded":"2026-02-22T22:31:07.146958+0000","last_fullsized":"2026-02-22T22:31:07.146958+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:57:09.391401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278449,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":114,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:06.655510+0000","last_change":"2026-02-22T22:24:37.193154+0000","last_active":"2026-02-22T22:31:06.655510+0000","last_peered":"2026-02-22T22:31:06.655510+0000","last_clean":"2026-02-22T22:31:06.655510+0000","last_became_active":"2026-02-22T22:24:34.315699+0000","last_became_peered":"2026-02-22T22:24:34.315699+0000","last_unstale":"2026-02-22T22:31:06.655510+0000","last_undegraded":"2026-02-22T22:31:06.655510+0000","last_fullsized":"2026-02-22T22:31:06.655510+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:06:14.131443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028952900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":114,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:06.655962+0000","last_change":"2026-02-22T22:24:37.193148+0000","last_active":"2026-02-22T22:31:06.655962+0000","last_peered":"2026-02-22T22:31:06.655962+0000","last_clean":"2026-02-22T22:31:06.655962+0000","last_became_active":"2026-02-22T22:24:34.315765+0000","last_became_peered":"2026-02-22T22:24:34.315765+0000","last_unstale":"2026-02-22T22:31:06.655962+0000","last_undegraded":"2026-02-22T22:31:06.655962+0000","last_fullsized":"2026-02-22T22:31:06.655962+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:23:11.125424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278349,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":112,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.936998+0000","last_change":"2026-02-22T22:24:36.318064+0000","last_active":"2026-02-22T22:29:58.936998+0000","last_peered":"2026-02-22T22:29:58.936998+0000","last_clean":"2026-02-22T22:29:58.936998+0000","last_became_active":"2026-02-22T22:24:34.315206+0000","last_became_peered":"2026-02-22T22:24:34.315206+0000","last_unstale":"2026-02-22T22:29:58.936998+0000","last_undegraded":"2026-02-22T22:29:58.936998+0000","last_fullsized":"2026-02-22T22:29:58.936998+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:48:25.916897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056075500000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":201,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.936555+0000","last_change":"2026-02-22T22:24:30.304133+0000","last_active":"2026-02-22T22:29:58.936555+0000","last_peered":"2026-02-22T22:29:58.936555+0000","last_clean":"2026-02-22T22:29:58.936555+0000","last_became_active":"2026-02-22T22:24:30.303257+0000","last_became_peered":"2026-02-22T22:24:30.303257+0000","last_unstale":"2026-02-22T22:29:58.936555+0000","last_undegraded":"2026-02-22T22:29:58.936555+0000","last_fullsized":"2026-02-22T22:29:58.936555+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_clean_scrub_stamp":"2026-02-22T22:24:29.295633+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:57:58.047355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738451,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":39948,"kb_used_data":504,"kb_used_omap":1,"kb_used_meta":39422,"kb_avail":94331892,"statfs":{"total":96636764160,"available":96595857408,"internally_reserved":0,"allocated":516096,"data_stored":237656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40368582},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738450,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":60124,"kb_used_data":24844,"kb_used_omap":1,"kb_used_meta":35262,"kb_avail":94311716,"statfs":{"total":96636764160,"available":96575197184,"internally_reserved":0,"allocated":25440256,"data_stored":25068380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36108742},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738450,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40528,"kb_used_data":1084,"kb_used_omap":1,"kb_used_meta":39422,"kb_avail":94331312,"statfs":{"total":96636764160,"available":96595263488,"internally_reserved":0,"allocated":1110016,"data_stored":828024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40368582},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:31:09.329 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T22:31:09.329 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:31:09.473 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:31:09.473 INFO:teuthology.orchestra.run.trial190.stderr:dumped all 2026-02-22T22:31:09.485 INFO:teuthology.orchestra.run.trial190.stdout:{"pg_ready":true,"pg_map":{"version":260,"stamp":"2026-02-22T22:31:08.333705+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":234,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":140600,"kb_used_data":26432,"kb_used_omap":4,"kb_used_meta":114107,"kb_avail":282974920,"statfs":{"total":289910292480,"available":289766318080,"internally_reserved":0,"allocated":27066368,"data_stored":26134060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":116845906},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1230467670,"num_objects":-476,"num_object_clones":0,"num_object_copies":-952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-476,"num_whiteouts":0,"num_read":-42682,"num_read_kb":-343005,"num_write":-5609,"num_write_kb":-30651,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-35,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001291"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":110,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.938303+0000","last_change":"2026-02-22T22:24:36.318108+0000","last_active":"2026-02-22T22:29:58.938303+0000","last_peered":"2026-02-22T22:29:58.938303+0000","last_clean":"2026-02-22T22:29:58.938303+0000","last_became_active":"2026-02-22T22:24:34.316382+0000","last_became_peered":"2026-02-22T22:24:34.316382+0000","last_unstale":"2026-02-22T22:29:58.938303+0000","last_undegraded":"2026-02-22T22:29:58.938303+0000","last_fullsized":"2026-02-22T22:29:58.938303+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:30:56.453533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057076399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":111,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.938800+0000","last_change":"2026-02-22T22:24:36.318553+0000","last_active":"2026-02-22T22:29:58.938800+0000","last_peered":"2026-02-22T22:29:58.938800+0000","last_clean":"2026-02-22T22:29:58.938800+0000","last_became_active":"2026-02-22T22:24:34.315465+0000","last_became_peered":"2026-02-22T22:24:34.315465+0000","last_unstale":"2026-02-22T22:29:58.938800+0000","last_undegraded":"2026-02-22T22:29:58.938800+0000","last_fullsized":"2026-02-22T22:29:58.938800+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:27:00.271117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00102524,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.938899+0000","last_change":"2026-02-22T22:24:36.318688+0000","last_active":"2026-02-22T22:29:58.938899+0000","last_peered":"2026-02-22T22:29:58.938899+0000","last_clean":"2026-02-22T22:29:58.938899+0000","last_became_active":"2026-02-22T22:24:34.316505+0000","last_became_peered":"2026-02-22T22:24:34.316505+0000","last_unstale":"2026-02-22T22:29:58.938899+0000","last_undegraded":"2026-02-22T22:29:58.938899+0000","last_fullsized":"2026-02-22T22:29:58.938899+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:54.581297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001096383,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":111,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.936914+0000","last_change":"2026-02-22T22:24:36.318328+0000","last_active":"2026-02-22T22:29:58.936914+0000","last_peered":"2026-02-22T22:29:58.936914+0000","last_clean":"2026-02-22T22:29:58.936914+0000","last_became_active":"2026-02-22T22:24:34.316336+0000","last_became_peered":"2026-02-22T22:24:34.316336+0000","last_unstale":"2026-02-22T22:29:58.936914+0000","last_undegraded":"2026-02-22T22:29:58.936914+0000","last_fullsized":"2026-02-22T22:29:58.936914+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:42:20.387771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079384900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":116,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.146958+0000","last_change":"2026-02-22T22:24:36.315539+0000","last_active":"2026-02-22T22:31:07.146958+0000","last_peered":"2026-02-22T22:31:07.146958+0000","last_clean":"2026-02-22T22:31:07.146958+0000","last_became_active":"2026-02-22T22:24:34.315494+0000","last_became_peered":"2026-02-22T22:24:34.315494+0000","last_unstale":"2026-02-22T22:31:07.146958+0000","last_undegraded":"2026-02-22T22:31:07.146958+0000","last_fullsized":"2026-02-22T22:31:07.146958+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:57:09.391401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278449,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":114,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:06.655510+0000","last_change":"2026-02-22T22:24:37.193154+0000","last_active":"2026-02-22T22:31:06.655510+0000","last_peered":"2026-02-22T22:31:06.655510+0000","last_clean":"2026-02-22T22:31:06.655510+0000","last_became_active":"2026-02-22T22:24:34.315699+0000","last_became_peered":"2026-02-22T22:24:34.315699+0000","last_unstale":"2026-02-22T22:31:06.655510+0000","last_undegraded":"2026-02-22T22:31:06.655510+0000","last_fullsized":"2026-02-22T22:31:06.655510+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:06:14.131443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028952900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":114,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:06.655962+0000","last_change":"2026-02-22T22:24:37.193148+0000","last_active":"2026-02-22T22:31:06.655962+0000","last_peered":"2026-02-22T22:31:06.655962+0000","last_clean":"2026-02-22T22:31:06.655962+0000","last_became_active":"2026-02-22T22:24:34.315765+0000","last_became_peered":"2026-02-22T22:24:34.315765+0000","last_unstale":"2026-02-22T22:31:06.655962+0000","last_undegraded":"2026-02-22T22:31:06.655962+0000","last_fullsized":"2026-02-22T22:31:06.655962+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:23:11.125424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278349,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":112,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.936998+0000","last_change":"2026-02-22T22:24:36.318064+0000","last_active":"2026-02-22T22:29:58.936998+0000","last_peered":"2026-02-22T22:29:58.936998+0000","last_clean":"2026-02-22T22:29:58.936998+0000","last_became_active":"2026-02-22T22:24:34.315206+0000","last_became_peered":"2026-02-22T22:24:34.315206+0000","last_unstale":"2026-02-22T22:29:58.936998+0000","last_undegraded":"2026-02-22T22:29:58.936998+0000","last_fullsized":"2026-02-22T22:29:58.936998+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:48:25.916897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056075500000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":201,"reported_epoch":60,"state":"active+clean","last_fresh":"2026-02-22T22:29:58.936555+0000","last_change":"2026-02-22T22:24:30.304133+0000","last_active":"2026-02-22T22:29:58.936555+0000","last_peered":"2026-02-22T22:29:58.936555+0000","last_clean":"2026-02-22T22:29:58.936555+0000","last_became_active":"2026-02-22T22:24:30.303257+0000","last_became_peered":"2026-02-22T22:24:30.303257+0000","last_unstale":"2026-02-22T22:29:58.936555+0000","last_undegraded":"2026-02-22T22:29:58.936555+0000","last_fullsized":"2026-02-22T22:29:58.936555+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_clean_scrub_stamp":"2026-02-22T22:24:29.295633+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:57:58.047355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738451,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":39948,"kb_used_data":504,"kb_used_omap":1,"kb_used_meta":39422,"kb_avail":94331892,"statfs":{"total":96636764160,"available":96595857408,"internally_reserved":0,"allocated":516096,"data_stored":237656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40368582},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738450,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":60124,"kb_used_data":24844,"kb_used_omap":1,"kb_used_meta":35262,"kb_avail":94311716,"statfs":{"total":96636764160,"available":96575197184,"internally_reserved":0,"allocated":25440256,"data_stored":25068380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36108742},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738450,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40528,"kb_used_data":1084,"kb_used_omap":1,"kb_used_meta":39422,"kb_avail":94331312,"statfs":{"total":96636764160,"available":96595263488,"internally_reserved":0,"allocated":1110016,"data_stored":828024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40368582},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:31:09.487 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:31:09.631 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:31:09.631 INFO:teuthology.orchestra.run.trial190.stdout:{"epoch":62,"fsid":"56258423-fe54-446e-ae04-77688092fba9","created":"2026-02-22T22:24:18.613141+0000","modified":"2026-02-22T22:31:07.649771+0000","last_up_change":"2026-02-22T22:24:27.289747+0000","last_in_change":"2026-02-22T22:24:19.135781+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:24:28.313150+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:24:32.728976+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"de5fe4e0-5530-4614-b01f-ba3053c3eff6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6800","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6801","nonce":1600625483}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6802","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6803","nonce":1600625483}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6806","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6807","nonce":1600625483}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6804","nonce":1600625483},{"type":"v1","addr":"10.20.193.190:6805","nonce":1600625483}]},"public_addr":"10.20.193.190:6801/1600625483","cluster_addr":"10.20.193.190:6803/1600625483","heartbeat_back_addr":"10.20.193.190:6807/1600625483","heartbeat_front_addr":"10.20.193.190:6805/1600625483","state":["exists","up"]},{"osd":1,"uuid":"05eb91fe-01a8-497c-aa22-da7f38c6ce3f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6816","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6817","nonce":587589839}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6818","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6819","nonce":587589839}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6822","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6823","nonce":587589839}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6820","nonce":587589839},{"type":"v1","addr":"10.20.193.190:6821","nonce":587589839}]},"public_addr":"10.20.193.190:6817/587589839","cluster_addr":"10.20.193.190:6819/587589839","heartbeat_back_addr":"10.20.193.190:6823/587589839","heartbeat_front_addr":"10.20.193.190:6821/587589839","state":["exists","up"]},{"osd":2,"uuid":"392cacdc-46dc-4995-a7ff-7eef33867ec2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6808","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6809","nonce":3480696925}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6810","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6811","nonce":3480696925}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6814","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6815","nonce":3480696925}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.190:6812","nonce":3480696925},{"type":"v1","addr":"10.20.193.190:6813","nonce":3480696925}]},"public_addr":"10.20.193.190:6809/3480696925","cluster_addr":"10.20.193.190:6811/3480696925","heartbeat_back_addr":"10.20.193.190:6815/3480696925","heartbeat_front_addr":"10.20.193.190:6813/3480696925","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:24:25.487155+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:24:25.926235+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:24:26.147908+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:31:10.643 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-22T22:31:10.643 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:31:10.729 INFO:teuthology.orchestra.run.trial190.stdout:{ 2026-02-22T22:31:10.729 INFO:teuthology.orchestra.run.trial190.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:31:10.729 INFO:teuthology.orchestra.run.trial190.stdout:} 2026-02-22T22:31:10.740 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-22T22:31:10.883 INFO:teuthology.orchestra.run.trial190.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T22:31:10.894 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-22T22:31:10.894 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:31:10.968 INFO:teuthology.orchestra.run.trial190.stdout:{ 2026-02-22T22:31:10.968 INFO:teuthology.orchestra.run.trial190.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:31:10.968 INFO:teuthology.orchestra.run.trial190.stdout:} 2026-02-22T22:31:10.977 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-22T22:31:11.119 INFO:teuthology.orchestra.run.trial190.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T22:31:11.130 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-22T22:31:11.130 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:31:11.204 INFO:teuthology.orchestra.run.trial190.stdout:{ 2026-02-22T22:31:11.204 INFO:teuthology.orchestra.run.trial190.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:31:11.204 INFO:teuthology.orchestra.run.trial190.stdout:} 2026-02-22T22:31:11.213 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-22T22:31:11.363 INFO:teuthology.orchestra.run.trial190.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T22:31:11.374 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:31:11.528 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:31:11.528 INFO:teuthology.orchestra.run.trial190.stderr:dumped all 2026-02-22T22:31:11.538 INFO:teuthology.orchestra.run.trial190.stdout:{"pg_ready":true,"pg_map":{"version":261,"stamp":"2026-02-22T22:31:10.333916+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":234,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":116848,"kb_used_data":2676,"kb_used_omap":4,"kb_used_meta":114107,"kb_avail":282998672,"statfs":{"total":289910292480,"available":289790640128,"internally_reserved":0,"allocated":2740224,"data_stored":1893723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":116845906},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1230467670,"num_objects":-476,"num_object_clones":0,"num_object_copies":-952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-476,"num_whiteouts":0,"num_read":-42682,"num_read_kb":-343005,"num_write":-5609,"num_write_kb":-30651,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-35,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001165"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":113,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.145922+0000","last_change":"2026-02-22T22:24:36.318108+0000","last_active":"2026-02-22T22:31:07.145922+0000","last_peered":"2026-02-22T22:31:07.145922+0000","last_clean":"2026-02-22T22:31:07.145922+0000","last_became_active":"2026-02-22T22:24:34.316382+0000","last_became_peered":"2026-02-22T22:24:34.316382+0000","last_unstale":"2026-02-22T22:31:07.145922+0000","last_undegraded":"2026-02-22T22:31:07.145922+0000","last_fullsized":"2026-02-22T22:31:07.145922+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:30:56.453533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057076399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":113,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.146113+0000","last_change":"2026-02-22T22:24:36.318553+0000","last_active":"2026-02-22T22:31:07.146113+0000","last_peered":"2026-02-22T22:31:07.146113+0000","last_clean":"2026-02-22T22:31:07.146113+0000","last_became_active":"2026-02-22T22:24:34.315465+0000","last_became_peered":"2026-02-22T22:24:34.315465+0000","last_unstale":"2026-02-22T22:31:07.146113+0000","last_undegraded":"2026-02-22T22:31:07.146113+0000","last_fullsized":"2026-02-22T22:31:07.146113+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:27:00.271117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00102524,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":113,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.147239+0000","last_change":"2026-02-22T22:24:36.318688+0000","last_active":"2026-02-22T22:31:07.147239+0000","last_peered":"2026-02-22T22:31:07.147239+0000","last_clean":"2026-02-22T22:31:07.147239+0000","last_became_active":"2026-02-22T22:24:34.316505+0000","last_became_peered":"2026-02-22T22:24:34.316505+0000","last_unstale":"2026-02-22T22:31:07.147239+0000","last_undegraded":"2026-02-22T22:31:07.147239+0000","last_fullsized":"2026-02-22T22:31:07.147239+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:58:54.581297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001096383,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":113,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.143129+0000","last_change":"2026-02-22T22:24:36.318328+0000","last_active":"2026-02-22T22:31:07.143129+0000","last_peered":"2026-02-22T22:31:07.143129+0000","last_clean":"2026-02-22T22:31:07.143129+0000","last_became_active":"2026-02-22T22:24:34.316336+0000","last_became_peered":"2026-02-22T22:24:34.316336+0000","last_unstale":"2026-02-22T22:31:07.143129+0000","last_undegraded":"2026-02-22T22:31:07.143129+0000","last_fullsized":"2026-02-22T22:31:07.143129+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:42:20.387771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079384900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":116,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.146958+0000","last_change":"2026-02-22T22:24:36.315539+0000","last_active":"2026-02-22T22:31:07.146958+0000","last_peered":"2026-02-22T22:31:07.146958+0000","last_clean":"2026-02-22T22:31:07.146958+0000","last_became_active":"2026-02-22T22:24:34.315494+0000","last_became_peered":"2026-02-22T22:24:34.315494+0000","last_unstale":"2026-02-22T22:31:07.146958+0000","last_undegraded":"2026-02-22T22:31:07.146958+0000","last_fullsized":"2026-02-22T22:31:07.146958+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:57:09.391401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278449,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":114,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:06.655510+0000","last_change":"2026-02-22T22:24:37.193154+0000","last_active":"2026-02-22T22:31:06.655510+0000","last_peered":"2026-02-22T22:31:06.655510+0000","last_clean":"2026-02-22T22:31:06.655510+0000","last_became_active":"2026-02-22T22:24:34.315699+0000","last_became_peered":"2026-02-22T22:24:34.315699+0000","last_unstale":"2026-02-22T22:31:06.655510+0000","last_undegraded":"2026-02-22T22:31:06.655510+0000","last_fullsized":"2026-02-22T22:31:06.655510+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:06:14.131443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028952900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":114,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:06.655962+0000","last_change":"2026-02-22T22:24:37.193148+0000","last_active":"2026-02-22T22:31:06.655962+0000","last_peered":"2026-02-22T22:31:06.655962+0000","last_clean":"2026-02-22T22:31:06.655962+0000","last_became_active":"2026-02-22T22:24:34.315765+0000","last_became_peered":"2026-02-22T22:24:34.315765+0000","last_unstale":"2026-02-22T22:31:06.655962+0000","last_undegraded":"2026-02-22T22:31:06.655962+0000","last_fullsized":"2026-02-22T22:31:06.655962+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:23:11.125424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278349,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":114,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.144722+0000","last_change":"2026-02-22T22:24:36.318064+0000","last_active":"2026-02-22T22:31:07.144722+0000","last_peered":"2026-02-22T22:31:07.144722+0000","last_clean":"2026-02-22T22:31:07.144722+0000","last_became_active":"2026-02-22T22:24:34.315206+0000","last_became_peered":"2026-02-22T22:24:34.315206+0000","last_unstale":"2026-02-22T22:31:07.144722+0000","last_undegraded":"2026-02-22T22:31:07.144722+0000","last_fullsized":"2026-02-22T22:31:07.144722+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:33.304406+0000","last_clean_scrub_stamp":"2026-02-22T22:24:33.304406+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:48:25.916897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056075500000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":203,"reported_epoch":61,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.144216+0000","last_change":"2026-02-22T22:24:30.304133+0000","last_active":"2026-02-22T22:31:07.144216+0000","last_peered":"2026-02-22T22:31:07.144216+0000","last_clean":"2026-02-22T22:31:07.144216+0000","last_became_active":"2026-02-22T22:24:30.303257+0000","last_became_peered":"2026-02-22T22:24:30.303257+0000","last_unstale":"2026-02-22T22:31:07.144216+0000","last_undegraded":"2026-02-22T22:31:07.144216+0000","last_fullsized":"2026-02-22T22:31:07.144216+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:24:29.295633+0000","last_clean_scrub_stamp":"2026-02-22T22:24:29.295633+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:57:58.047355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738451,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":39948,"kb_used_data":504,"kb_used_omap":1,"kb_used_meta":39422,"kb_avail":94331892,"statfs":{"total":96636764160,"available":96595857408,"internally_reserved":0,"allocated":516096,"data_stored":237656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40368582},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738451,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36372,"kb_used_data":1088,"kb_used_omap":1,"kb_used_meta":35262,"kb_avail":94335468,"statfs":{"total":96636764160,"available":96599519232,"internally_reserved":0,"allocated":1114112,"data_stored":828043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36108742},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738450,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40528,"kb_used_data":1084,"kb_used_omap":1,"kb_used_meta":39422,"kb_avail":94331312,"statfs":{"total":96636764160,"available":96595263488,"internally_reserved":0,"allocated":1110016,"data_stored":828024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40368582},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:31:11.541 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T22:24:33.304406+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=24, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=9, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:11.541 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T22:24:33.304406+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=24, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=9, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:11.541 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T22:24:33.304406+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=24, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=9, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:11.541 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T22:24:33.304406+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=24, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=9, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:11.541 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T22:24:33.304406+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=24, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=9, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:11.541 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T22:24:33.304406+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=24, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=9, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:11.541 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T22:24:33.304406+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=24, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=9, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:11.541 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T22:24:33.304406+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=24, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=9, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:11.542 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T22:24:29.295633+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=24, tm_sec=29, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=9, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:11.542 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T22:31:31.542 DEBUG:teuthology.orchestra.run.trial190:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:31:31.692 INFO:teuthology.orchestra.run.trial190.stdout: 2026-02-22T22:31:31.693 INFO:teuthology.orchestra.run.trial190.stderr:dumped all 2026-02-22T22:31:31.702 INFO:teuthology.orchestra.run.trial190.stdout:{"pg_ready":true,"pg_map":{"version":271,"stamp":"2026-02-22T22:31:30.335521+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":116864,"kb_used_data":2704,"kb_used_omap":4,"kb_used_meta":114107,"kb_avail":282998656,"statfs":{"total":289910292480,"available":289790623744,"internally_reserved":0,"allocated":2768896,"data_stored":1904569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":116845906},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000913"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":124,"reported_epoch":62,"state":"active+clean","last_fresh":"2026-02-22T22:31:15.128574+0000","last_change":"2026-02-22T22:31:15.128589+0000","last_active":"2026-02-22T22:31:15.128574+0000","last_peered":"2026-02-22T22:31:15.128574+0000","last_clean":"2026-02-22T22:31:15.128574+0000","last_became_active":"2026-02-22T22:24:34.316382+0000","last_became_peered":"2026-02-22T22:24:34.316382+0000","last_unstale":"2026-02-22T22:31:15.128574+0000","last_undegraded":"2026-02-22T22:31:15.128574+0000","last_fullsized":"2026-02-22T22:31:15.128574+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:15.128511+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:15.128511+0000","last_clean_scrub_stamp":"2026-02-22T22:31:15.128511+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:34:49.146697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00057076399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":124,"reported_epoch":62,"state":"active+clean","last_fresh":"2026-02-22T22:31:13.093219+0000","last_change":"2026-02-22T22:31:13.093237+0000","last_active":"2026-02-22T22:31:13.093219+0000","last_peered":"2026-02-22T22:31:13.093219+0000","last_clean":"2026-02-22T22:31:13.093219+0000","last_became_active":"2026-02-22T22:24:34.315465+0000","last_became_peered":"2026-02-22T22:24:34.315465+0000","last_unstale":"2026-02-22T22:31:13.093219+0000","last_undegraded":"2026-02-22T22:31:13.093219+0000","last_fullsized":"2026-02-22T22:31:13.093219+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:13.093149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:13.093149+0000","last_clean_scrub_stamp":"2026-02-22T22:31:13.093149+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:35:52.260436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00102524,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":124,"reported_epoch":62,"state":"active+clean","last_fresh":"2026-02-22T22:31:17.059899+0000","last_change":"2026-02-22T22:31:17.059913+0000","last_active":"2026-02-22T22:31:17.059899+0000","last_peered":"2026-02-22T22:31:17.059899+0000","last_clean":"2026-02-22T22:31:17.059899+0000","last_became_active":"2026-02-22T22:24:34.316505+0000","last_became_peered":"2026-02-22T22:24:34.316505+0000","last_unstale":"2026-02-22T22:31:17.059899+0000","last_undegraded":"2026-02-22T22:31:17.059899+0000","last_fullsized":"2026-02-22T22:31:17.059899+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:17.059833+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:17.059833+0000","last_clean_scrub_stamp":"2026-02-22T22:31:17.059833+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:39:05.033554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001096383,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":124,"reported_epoch":62,"state":"active+clean","last_fresh":"2026-02-22T22:31:16.101166+0000","last_change":"2026-02-22T22:31:16.101196+0000","last_active":"2026-02-22T22:31:16.101166+0000","last_peered":"2026-02-22T22:31:16.101166+0000","last_clean":"2026-02-22T22:31:16.101166+0000","last_became_active":"2026-02-22T22:24:34.316336+0000","last_became_peered":"2026-02-22T22:24:34.316336+0000","last_unstale":"2026-02-22T22:31:16.101166+0000","last_undegraded":"2026-02-22T22:31:16.101166+0000","last_fullsized":"2026-02-22T22:31:16.101166+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:16.101085+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:16.101085+0000","last_clean_scrub_stamp":"2026-02-22T22:31:16.101085+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:00:30.423876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079384900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"62'3","reported_seq":128,"reported_epoch":62,"state":"active+clean","last_fresh":"2026-02-22T22:31:11.260795+0000","last_change":"2026-02-22T22:31:11.260819+0000","last_active":"2026-02-22T22:31:11.260795+0000","last_peered":"2026-02-22T22:31:11.260795+0000","last_clean":"2026-02-22T22:31:11.260795+0000","last_became_active":"2026-02-22T22:24:34.315494+0000","last_became_peered":"2026-02-22T22:24:34.315494+0000","last_unstale":"2026-02-22T22:31:11.260795+0000","last_undegraded":"2026-02-22T22:31:11.260795+0000","last_fullsized":"2026-02-22T22:31:11.260795+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"62'3","last_scrub_stamp":"2026-02-22T22:31:11.260733+0000","last_deep_scrub":"62'3","last_deep_scrub_stamp":"2026-02-22T22:31:11.260733+0000","last_clean_scrub_stamp":"2026-02-22T22:31:11.260733+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:00:21.355262+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.000278449,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":125,"reported_epoch":62,"state":"active+clean","last_fresh":"2026-02-22T22:31:13.220143+0000","last_change":"2026-02-22T22:31:13.220158+0000","last_active":"2026-02-22T22:31:13.220143+0000","last_peered":"2026-02-22T22:31:13.220143+0000","last_clean":"2026-02-22T22:31:13.220143+0000","last_became_active":"2026-02-22T22:24:34.315699+0000","last_became_peered":"2026-02-22T22:24:34.315699+0000","last_unstale":"2026-02-22T22:31:13.220143+0000","last_undegraded":"2026-02-22T22:31:13.220143+0000","last_fullsized":"2026-02-22T22:31:13.220143+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:13.220083+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:13.220083+0000","last_clean_scrub_stamp":"2026-02-22T22:31:13.220083+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:15:09.185039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028952900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":125,"reported_epoch":62,"state":"active+clean","last_fresh":"2026-02-22T22:31:12.268388+0000","last_change":"2026-02-22T22:31:12.268412+0000","last_active":"2026-02-22T22:31:12.268388+0000","last_peered":"2026-02-22T22:31:12.268388+0000","last_clean":"2026-02-22T22:31:12.268388+0000","last_became_active":"2026-02-22T22:24:34.315765+0000","last_became_peered":"2026-02-22T22:24:34.315765+0000","last_unstale":"2026-02-22T22:31:12.268388+0000","last_undegraded":"2026-02-22T22:31:12.268388+0000","last_fullsized":"2026-02-22T22:31:12.268388+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:12.268278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:12.268278+0000","last_clean_scrub_stamp":"2026-02-22T22:31:12.268278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:14:03.381324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000278349,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"62'2","reported_seq":126,"reported_epoch":62,"state":"active+clean","last_fresh":"2026-02-22T22:31:14.099121+0000","last_change":"2026-02-22T22:31:14.099178+0000","last_active":"2026-02-22T22:31:14.099121+0000","last_peered":"2026-02-22T22:31:14.099121+0000","last_clean":"2026-02-22T22:31:14.099121+0000","last_became_active":"2026-02-22T22:24:34.315206+0000","last_became_peered":"2026-02-22T22:24:34.315206+0000","last_unstale":"2026-02-22T22:31:14.099121+0000","last_undegraded":"2026-02-22T22:31:14.099121+0000","last_fullsized":"2026-02-22T22:31:14.099121+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"62'2","last_scrub_stamp":"2026-02-22T22:31:14.099028+0000","last_deep_scrub":"62'2","last_deep_scrub_stamp":"2026-02-22T22:31:14.099028+0000","last_clean_scrub_stamp":"2026-02-22T22:31:14.099028+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:57:40.629289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056075500000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"62'74","reported_seq":216,"reported_epoch":62,"state":"active+clean","last_fresh":"2026-02-22T22:31:12.130169+0000","last_change":"2026-02-22T22:31:12.130197+0000","last_active":"2026-02-22T22:31:12.130169+0000","last_peered":"2026-02-22T22:31:12.130169+0000","last_clean":"2026-02-22T22:31:12.130169+0000","last_became_active":"2026-02-22T22:24:30.303257+0000","last_became_peered":"2026-02-22T22:24:30.303257+0000","last_unstale":"2026-02-22T22:31:12.130169+0000","last_undegraded":"2026-02-22T22:31:12.130169+0000","last_fullsized":"2026-02-22T22:31:12.130169+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"62'74","last_scrub_stamp":"2026-02-22T22:31:12.129931+0000","last_deep_scrub":"62'74","last_deep_scrub_stamp":"2026-02-22T22:31:12.129931+0000","last_clean_scrub_stamp":"2026-02-22T22:31:12.129931+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:42:51.294680+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738455,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":39952,"kb_used_data":512,"kb_used_omap":1,"kb_used_meta":39422,"kb_avail":94331888,"statfs":{"total":96636764160,"available":96595853312,"internally_reserved":0,"allocated":524288,"data_stored":241265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40368582},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738455,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":36376,"kb_used_data":1096,"kb_used_omap":1,"kb_used_meta":35262,"kb_avail":94335464,"statfs":{"total":96636764160,"available":96599515136,"internally_reserved":0,"allocated":1122304,"data_stored":831652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":36108742},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738454,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":40536,"kb_used_data":1096,"kb_used_omap":1,"kb_used_meta":39422,"kb_avail":94331304,"statfs":{"total":96636764160,"available":96595255296,"internally_reserved":0,"allocated":1122304,"data_stored":831652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":40368582},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:31:31.703 DEBUG:teuthology.orchestra.run.trial190:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-22T22:31:31.857 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T22:31:31.857 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T22:31:31.858 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-22T22:31:31.858 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:31:31.940 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-22T22:31:31.940 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-22T22:31:31.940 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:31:32.012 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-22T22:31:32.012 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-22T22:31:32.012 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:31:32.092 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-22T22:31:32.092 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T22:31:32.092 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-22T22:31:32.092 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:31:32.134 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-22T22:31:32.134 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T22:31:32.135 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-22T22:31:32.135 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:31:32.148 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-22T22:31:32.148 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T22:31:32.148 DEBUG:teuthology.orchestra.run.trial190:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T22:31:32.200 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:31:32.200 DEBUG:teuthology.orchestra.run.trial190:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-22T22:31:32.259 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:31:32.259 DEBUG:teuthology.orchestra.run.trial190:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-22T22:31:32.312 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:31:32.312 DEBUG:teuthology.orchestra.run.trial190:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-22T22:31:32.367 INFO:tasks.ceph:Archiving mon data... 2026-02-22T22:31:32.371 DEBUG:teuthology.misc:Transferring archived files from trial190:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215/data/cluster2.mon.a.tgz 2026-02-22T22:31:32.371 DEBUG:teuthology.orchestra.run.trial190:> mktemp 2026-02-22T22:31:32.411 INFO:teuthology.orchestra.run.trial190.stdout:/tmp/tmp.iipxmgXY1h 2026-02-22T22:31:32.411 DEBUG:teuthology.orchestra.run.trial190:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.iipxmgXY1h 2026-02-22T22:31:32.498 DEBUG:teuthology.orchestra.run.trial190:> sudo chmod 0666 /tmp/tmp.iipxmgXY1h 2026-02-22T22:31:32.556 DEBUG:teuthology.orchestra.remote:trial190:/tmp/tmp.iipxmgXY1h is 293KB 2026-02-22T22:31:32.609 DEBUG:teuthology.orchestra.run.trial190:> rm -fr /tmp/tmp.iipxmgXY1h 2026-02-22T22:31:32.613 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T22:31:32.614 DEBUG:teuthology.orchestra.run.trial103:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:31:32.616 DEBUG:teuthology.orchestra.run.trial190:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:31:32.663 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T22:31:32.676 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T22:31:32.676 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:31:32.826 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:31:32.826 INFO:teuthology.orchestra.run.trial103.stderr:dumped all 2026-02-22T22:31:32.835 INFO:teuthology.orchestra.run.trial103.stdout:{"pg_ready":true,"pg_map":{"version":301,"stamp":"2026-02-22T22:31:31.170230+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":88032,"kb_used_data":2980,"kb_used_omap":4,"kb_used_meta":84987,"kb_avail":283027488,"statfs":{"total":289910292480,"available":289820147712,"internally_reserved":0,"allocated":3051520,"data_stored":2026630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":87027027},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000872"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441291+0000","last_change":"2026-02-22T22:23:59.140328+0000","last_active":"2026-02-22T22:31:07.441291+0000","last_peered":"2026-02-22T22:31:07.441291+0000","last_clean":"2026-02-22T22:31:07.441291+0000","last_became_active":"2026-02-22T22:23:57.137067+0000","last_became_peered":"2026-02-22T22:23:57.137067+0000","last_unstale":"2026-02-22T22:31:07.441291+0000","last_undegraded":"2026-02-22T22:31:07.441291+0000","last_fullsized":"2026-02-22T22:31:07.441291+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:01:31.446177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061824999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441310+0000","last_change":"2026-02-22T22:23:59.140089+0000","last_active":"2026-02-22T22:31:07.441310+0000","last_peered":"2026-02-22T22:31:07.441310+0000","last_clean":"2026-02-22T22:31:07.441310+0000","last_became_active":"2026-02-22T22:23:57.137050+0000","last_became_peered":"2026-02-22T22:23:57.137050+0000","last_unstale":"2026-02-22T22:31:07.441310+0000","last_undegraded":"2026-02-22T22:31:07.441310+0000","last_fullsized":"2026-02-22T22:31:07.441310+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:03:18.154802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041303899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.436984+0000","last_change":"2026-02-22T22:23:59.140496+0000","last_active":"2026-02-22T22:31:07.436984+0000","last_peered":"2026-02-22T22:31:07.436984+0000","last_clean":"2026-02-22T22:31:07.436984+0000","last_became_active":"2026-02-22T22:23:57.136991+0000","last_became_peered":"2026-02-22T22:23:57.136991+0000","last_unstale":"2026-02-22T22:31:07.436984+0000","last_undegraded":"2026-02-22T22:31:07.436984+0000","last_fullsized":"2026-02-22T22:31:07.436984+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:14:39.011925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078158500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.440762+0000","last_change":"2026-02-22T22:23:59.140094+0000","last_active":"2026-02-22T22:31:07.440762+0000","last_peered":"2026-02-22T22:31:07.440762+0000","last_clean":"2026-02-22T22:31:07.440762+0000","last_became_active":"2026-02-22T22:23:57.136930+0000","last_became_peered":"2026-02-22T22:23:57.136930+0000","last_unstale":"2026-02-22T22:31:07.440762+0000","last_undegraded":"2026-02-22T22:31:07.440762+0000","last_fullsized":"2026-02-22T22:31:07.440762+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:49:52.109397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040949200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":133,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.138280+0000","last_change":"2026-02-22T22:23:59.138515+0000","last_active":"2026-02-22T22:31:07.138280+0000","last_peered":"2026-02-22T22:31:07.138280+0000","last_clean":"2026-02-22T22:31:07.138280+0000","last_became_active":"2026-02-22T22:23:57.136352+0000","last_became_peered":"2026-02-22T22:23:57.136352+0000","last_unstale":"2026-02-22T22:31:07.138280+0000","last_undegraded":"2026-02-22T22:31:07.138280+0000","last_fullsized":"2026-02-22T22:31:07.138280+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:56.872656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061167800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":132,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.444375+0000","last_change":"2026-02-22T22:23:59.136866+0000","last_active":"2026-02-22T22:31:07.444375+0000","last_peered":"2026-02-22T22:31:07.444375+0000","last_clean":"2026-02-22T22:31:07.444375+0000","last_became_active":"2026-02-22T22:23:57.136381+0000","last_became_peered":"2026-02-22T22:23:57.136381+0000","last_unstale":"2026-02-22T22:31:07.444375+0000","last_undegraded":"2026-02-22T22:31:07.444375+0000","last_fullsized":"2026-02-22T22:31:07.444375+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:47:13.328371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023819299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":132,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.445875+0000","last_change":"2026-02-22T22:23:59.136809+0000","last_active":"2026-02-22T22:31:07.445875+0000","last_peered":"2026-02-22T22:31:07.445875+0000","last_clean":"2026-02-22T22:31:07.445875+0000","last_became_active":"2026-02-22T22:23:57.136128+0000","last_became_peered":"2026-02-22T22:23:57.136128+0000","last_unstale":"2026-02-22T22:31:07.445875+0000","last_undegraded":"2026-02-22T22:31:07.445875+0000","last_fullsized":"2026-02-22T22:31:07.445875+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:33.645236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000213277,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":135,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441631+0000","last_change":"2026-02-22T22:23:59.140296+0000","last_active":"2026-02-22T22:31:07.441631+0000","last_peered":"2026-02-22T22:31:07.441631+0000","last_clean":"2026-02-22T22:31:07.441631+0000","last_became_active":"2026-02-22T22:23:57.136046+0000","last_became_peered":"2026-02-22T22:23:57.136046+0000","last_unstale":"2026-02-22T22:31:07.441631+0000","last_undegraded":"2026-02-22T22:31:07.441631+0000","last_fullsized":"2026-02-22T22:31:07.441631+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:27:57.025803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000595328,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":225,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441815+0000","last_change":"2026-02-22T22:23:53.494316+0000","last_active":"2026-02-22T22:31:07.441815+0000","last_peered":"2026-02-22T22:31:07.441815+0000","last_clean":"2026-02-22T22:31:07.441815+0000","last_became_active":"2026-02-22T22:23:53.493528+0000","last_became_peered":"2026-02-22T22:23:53.493528+0000","last_unstale":"2026-02-22T22:31:07.441815+0000","last_undegraded":"2026-02-22T22:31:07.441815+0000","last_fullsized":"2026-02-22T22:31:07.441815+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_clean_scrub_stamp":"2026-02-22T22:23:52.116710+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:04:37.859993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705759,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27568,"kb_used_data":604,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344272,"statfs":{"total":96636764160,"available":96608534528,"internally_reserved":0,"allocated":618496,"data_stored":281952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705759,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32312,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339528,"statfs":{"total":96636764160,"available":96603676672,"internally_reserved":0,"allocated":1216512,"data_stored":872339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738464,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28152,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94343688,"statfs":{"total":96636764160,"available":96607936512,"internally_reserved":0,"allocated":1216512,"data_stored":872339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:31:32.837 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:31:32.985 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:31:32.985 INFO:teuthology.orchestra.run.trial103.stderr:dumped all 2026-02-22T22:31:32.995 INFO:teuthology.orchestra.run.trial103.stdout:{"pg_ready":true,"pg_map":{"version":301,"stamp":"2026-02-22T22:31:31.170230+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":88032,"kb_used_data":2980,"kb_used_omap":4,"kb_used_meta":84987,"kb_avail":283027488,"statfs":{"total":289910292480,"available":289820147712,"internally_reserved":0,"allocated":3051520,"data_stored":2026630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":87027027},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000872"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441291+0000","last_change":"2026-02-22T22:23:59.140328+0000","last_active":"2026-02-22T22:31:07.441291+0000","last_peered":"2026-02-22T22:31:07.441291+0000","last_clean":"2026-02-22T22:31:07.441291+0000","last_became_active":"2026-02-22T22:23:57.137067+0000","last_became_peered":"2026-02-22T22:23:57.137067+0000","last_unstale":"2026-02-22T22:31:07.441291+0000","last_undegraded":"2026-02-22T22:31:07.441291+0000","last_fullsized":"2026-02-22T22:31:07.441291+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:01:31.446177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061824999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441310+0000","last_change":"2026-02-22T22:23:59.140089+0000","last_active":"2026-02-22T22:31:07.441310+0000","last_peered":"2026-02-22T22:31:07.441310+0000","last_clean":"2026-02-22T22:31:07.441310+0000","last_became_active":"2026-02-22T22:23:57.137050+0000","last_became_peered":"2026-02-22T22:23:57.137050+0000","last_unstale":"2026-02-22T22:31:07.441310+0000","last_undegraded":"2026-02-22T22:31:07.441310+0000","last_fullsized":"2026-02-22T22:31:07.441310+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:03:18.154802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041303899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.436984+0000","last_change":"2026-02-22T22:23:59.140496+0000","last_active":"2026-02-22T22:31:07.436984+0000","last_peered":"2026-02-22T22:31:07.436984+0000","last_clean":"2026-02-22T22:31:07.436984+0000","last_became_active":"2026-02-22T22:23:57.136991+0000","last_became_peered":"2026-02-22T22:23:57.136991+0000","last_unstale":"2026-02-22T22:31:07.436984+0000","last_undegraded":"2026-02-22T22:31:07.436984+0000","last_fullsized":"2026-02-22T22:31:07.436984+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:14:39.011925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078158500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.440762+0000","last_change":"2026-02-22T22:23:59.140094+0000","last_active":"2026-02-22T22:31:07.440762+0000","last_peered":"2026-02-22T22:31:07.440762+0000","last_clean":"2026-02-22T22:31:07.440762+0000","last_became_active":"2026-02-22T22:23:57.136930+0000","last_became_peered":"2026-02-22T22:23:57.136930+0000","last_unstale":"2026-02-22T22:31:07.440762+0000","last_undegraded":"2026-02-22T22:31:07.440762+0000","last_fullsized":"2026-02-22T22:31:07.440762+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:49:52.109397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040949200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":133,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.138280+0000","last_change":"2026-02-22T22:23:59.138515+0000","last_active":"2026-02-22T22:31:07.138280+0000","last_peered":"2026-02-22T22:31:07.138280+0000","last_clean":"2026-02-22T22:31:07.138280+0000","last_became_active":"2026-02-22T22:23:57.136352+0000","last_became_peered":"2026-02-22T22:23:57.136352+0000","last_unstale":"2026-02-22T22:31:07.138280+0000","last_undegraded":"2026-02-22T22:31:07.138280+0000","last_fullsized":"2026-02-22T22:31:07.138280+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:56.872656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061167800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":132,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.444375+0000","last_change":"2026-02-22T22:23:59.136866+0000","last_active":"2026-02-22T22:31:07.444375+0000","last_peered":"2026-02-22T22:31:07.444375+0000","last_clean":"2026-02-22T22:31:07.444375+0000","last_became_active":"2026-02-22T22:23:57.136381+0000","last_became_peered":"2026-02-22T22:23:57.136381+0000","last_unstale":"2026-02-22T22:31:07.444375+0000","last_undegraded":"2026-02-22T22:31:07.444375+0000","last_fullsized":"2026-02-22T22:31:07.444375+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:47:13.328371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023819299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":132,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.445875+0000","last_change":"2026-02-22T22:23:59.136809+0000","last_active":"2026-02-22T22:31:07.445875+0000","last_peered":"2026-02-22T22:31:07.445875+0000","last_clean":"2026-02-22T22:31:07.445875+0000","last_became_active":"2026-02-22T22:23:57.136128+0000","last_became_peered":"2026-02-22T22:23:57.136128+0000","last_unstale":"2026-02-22T22:31:07.445875+0000","last_undegraded":"2026-02-22T22:31:07.445875+0000","last_fullsized":"2026-02-22T22:31:07.445875+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:33.645236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000213277,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":135,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441631+0000","last_change":"2026-02-22T22:23:59.140296+0000","last_active":"2026-02-22T22:31:07.441631+0000","last_peered":"2026-02-22T22:31:07.441631+0000","last_clean":"2026-02-22T22:31:07.441631+0000","last_became_active":"2026-02-22T22:23:57.136046+0000","last_became_peered":"2026-02-22T22:23:57.136046+0000","last_unstale":"2026-02-22T22:31:07.441631+0000","last_undegraded":"2026-02-22T22:31:07.441631+0000","last_fullsized":"2026-02-22T22:31:07.441631+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:27:57.025803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000595328,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":225,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441815+0000","last_change":"2026-02-22T22:23:53.494316+0000","last_active":"2026-02-22T22:31:07.441815+0000","last_peered":"2026-02-22T22:31:07.441815+0000","last_clean":"2026-02-22T22:31:07.441815+0000","last_became_active":"2026-02-22T22:23:53.493528+0000","last_became_peered":"2026-02-22T22:23:53.493528+0000","last_unstale":"2026-02-22T22:31:07.441815+0000","last_undegraded":"2026-02-22T22:31:07.441815+0000","last_fullsized":"2026-02-22T22:31:07.441815+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_clean_scrub_stamp":"2026-02-22T22:23:52.116710+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:04:37.859993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705759,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27568,"kb_used_data":604,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344272,"statfs":{"total":96636764160,"available":96608534528,"internally_reserved":0,"allocated":618496,"data_stored":281952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705759,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32312,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339528,"statfs":{"total":96636764160,"available":96603676672,"internally_reserved":0,"allocated":1216512,"data_stored":872339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738464,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28152,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94343688,"statfs":{"total":96636764160,"available":96607936512,"internally_reserved":0,"allocated":1216512,"data_stored":872339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:31:32.996 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T22:31:32.996 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:31:33.144 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:31:33.144 INFO:teuthology.orchestra.run.trial103.stderr:dumped all 2026-02-22T22:31:33.153 INFO:teuthology.orchestra.run.trial103.stdout:{"pg_ready":true,"pg_map":{"version":301,"stamp":"2026-02-22T22:31:31.170230+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":88032,"kb_used_data":2980,"kb_used_omap":4,"kb_used_meta":84987,"kb_avail":283027488,"statfs":{"total":289910292480,"available":289820147712,"internally_reserved":0,"allocated":3051520,"data_stored":2026630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":87027027},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000872"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441291+0000","last_change":"2026-02-22T22:23:59.140328+0000","last_active":"2026-02-22T22:31:07.441291+0000","last_peered":"2026-02-22T22:31:07.441291+0000","last_clean":"2026-02-22T22:31:07.441291+0000","last_became_active":"2026-02-22T22:23:57.137067+0000","last_became_peered":"2026-02-22T22:23:57.137067+0000","last_unstale":"2026-02-22T22:31:07.441291+0000","last_undegraded":"2026-02-22T22:31:07.441291+0000","last_fullsized":"2026-02-22T22:31:07.441291+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:01:31.446177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061824999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441310+0000","last_change":"2026-02-22T22:23:59.140089+0000","last_active":"2026-02-22T22:31:07.441310+0000","last_peered":"2026-02-22T22:31:07.441310+0000","last_clean":"2026-02-22T22:31:07.441310+0000","last_became_active":"2026-02-22T22:23:57.137050+0000","last_became_peered":"2026-02-22T22:23:57.137050+0000","last_unstale":"2026-02-22T22:31:07.441310+0000","last_undegraded":"2026-02-22T22:31:07.441310+0000","last_fullsized":"2026-02-22T22:31:07.441310+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:03:18.154802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041303899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.436984+0000","last_change":"2026-02-22T22:23:59.140496+0000","last_active":"2026-02-22T22:31:07.436984+0000","last_peered":"2026-02-22T22:31:07.436984+0000","last_clean":"2026-02-22T22:31:07.436984+0000","last_became_active":"2026-02-22T22:23:57.136991+0000","last_became_peered":"2026-02-22T22:23:57.136991+0000","last_unstale":"2026-02-22T22:31:07.436984+0000","last_undegraded":"2026-02-22T22:31:07.436984+0000","last_fullsized":"2026-02-22T22:31:07.436984+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:14:39.011925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078158500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":134,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.440762+0000","last_change":"2026-02-22T22:23:59.140094+0000","last_active":"2026-02-22T22:31:07.440762+0000","last_peered":"2026-02-22T22:31:07.440762+0000","last_clean":"2026-02-22T22:31:07.440762+0000","last_became_active":"2026-02-22T22:23:57.136930+0000","last_became_peered":"2026-02-22T22:23:57.136930+0000","last_unstale":"2026-02-22T22:31:07.440762+0000","last_undegraded":"2026-02-22T22:31:07.440762+0000","last_fullsized":"2026-02-22T22:31:07.440762+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:49:52.109397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040949200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":133,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.138280+0000","last_change":"2026-02-22T22:23:59.138515+0000","last_active":"2026-02-22T22:31:07.138280+0000","last_peered":"2026-02-22T22:31:07.138280+0000","last_clean":"2026-02-22T22:31:07.138280+0000","last_became_active":"2026-02-22T22:23:57.136352+0000","last_became_peered":"2026-02-22T22:23:57.136352+0000","last_unstale":"2026-02-22T22:31:07.138280+0000","last_undegraded":"2026-02-22T22:31:07.138280+0000","last_fullsized":"2026-02-22T22:31:07.138280+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:56.872656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061167800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":132,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.444375+0000","last_change":"2026-02-22T22:23:59.136866+0000","last_active":"2026-02-22T22:31:07.444375+0000","last_peered":"2026-02-22T22:31:07.444375+0000","last_clean":"2026-02-22T22:31:07.444375+0000","last_became_active":"2026-02-22T22:23:57.136381+0000","last_became_peered":"2026-02-22T22:23:57.136381+0000","last_unstale":"2026-02-22T22:31:07.444375+0000","last_undegraded":"2026-02-22T22:31:07.444375+0000","last_fullsized":"2026-02-22T22:31:07.444375+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:47:13.328371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023819299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":132,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.445875+0000","last_change":"2026-02-22T22:23:59.136809+0000","last_active":"2026-02-22T22:31:07.445875+0000","last_peered":"2026-02-22T22:31:07.445875+0000","last_clean":"2026-02-22T22:31:07.445875+0000","last_became_active":"2026-02-22T22:23:57.136128+0000","last_became_peered":"2026-02-22T22:23:57.136128+0000","last_unstale":"2026-02-22T22:31:07.445875+0000","last_undegraded":"2026-02-22T22:31:07.445875+0000","last_fullsized":"2026-02-22T22:31:07.445875+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:33.645236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000213277,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":135,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441631+0000","last_change":"2026-02-22T22:23:59.140296+0000","last_active":"2026-02-22T22:31:07.441631+0000","last_peered":"2026-02-22T22:31:07.441631+0000","last_clean":"2026-02-22T22:31:07.441631+0000","last_became_active":"2026-02-22T22:23:57.136046+0000","last_became_peered":"2026-02-22T22:23:57.136046+0000","last_unstale":"2026-02-22T22:31:07.441631+0000","last_undegraded":"2026-02-22T22:31:07.441631+0000","last_fullsized":"2026-02-22T22:31:07.441631+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:27:57.025803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000595328,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":225,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441815+0000","last_change":"2026-02-22T22:23:53.494316+0000","last_active":"2026-02-22T22:31:07.441815+0000","last_peered":"2026-02-22T22:31:07.441815+0000","last_clean":"2026-02-22T22:31:07.441815+0000","last_became_active":"2026-02-22T22:23:53.493528+0000","last_became_peered":"2026-02-22T22:23:53.493528+0000","last_unstale":"2026-02-22T22:31:07.441815+0000","last_undegraded":"2026-02-22T22:31:07.441815+0000","last_fullsized":"2026-02-22T22:31:07.441815+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_clean_scrub_stamp":"2026-02-22T22:23:52.116710+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:04:37.859993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705759,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27568,"kb_used_data":604,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344272,"statfs":{"total":96636764160,"available":96608534528,"internally_reserved":0,"allocated":618496,"data_stored":281952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705759,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32312,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339528,"statfs":{"total":96636764160,"available":96603676672,"internally_reserved":0,"allocated":1216512,"data_stored":872339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738464,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28152,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94343688,"statfs":{"total":96636764160,"available":96607936512,"internally_reserved":0,"allocated":1216512,"data_stored":872339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:31:33.153 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:31:33.306 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:31:33.306 INFO:teuthology.orchestra.run.trial103.stdout:{"epoch":72,"fsid":"a7533019-ce65-46ba-9683-fc039f8c54d6","created":"2026-02-22T22:23:40.945517+0000","modified":"2026-02-22T22:31:07.125004+0000","last_up_change":"2026-02-22T22:23:50.110835+0000","last_in_change":"2026-02-22T22:23:41.556987+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:23:51.138178+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:23:55.134505+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"08a89853-1713-4d9a-9778-de259bc9f405","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6810","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6811","nonce":3539595539}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6812","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6813","nonce":3539595539}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6816","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6817","nonce":3539595539}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6814","nonce":3539595539},{"type":"v1","addr":"10.20.193.103:6815","nonce":3539595539}]},"public_addr":"10.20.193.103:6811/3539595539","cluster_addr":"10.20.193.103:6813/3539595539","heartbeat_back_addr":"10.20.193.103:6817/3539595539","heartbeat_front_addr":"10.20.193.103:6815/3539595539","state":["exists","up"]},{"osd":1,"uuid":"dae311b1-56ae-4e70-b128-073cdb3bb612","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6802","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6803","nonce":3310548690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6804","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6805","nonce":3310548690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6808","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6809","nonce":3310548690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6806","nonce":3310548690},{"type":"v1","addr":"10.20.193.103:6807","nonce":3310548690}]},"public_addr":"10.20.193.103:6803/3310548690","cluster_addr":"10.20.193.103:6805/3310548690","heartbeat_back_addr":"10.20.193.103:6809/3310548690","heartbeat_front_addr":"10.20.193.103:6807/3310548690","state":["exists","up"]},{"osd":2,"uuid":"f3ae4fde-cfef-46fb-998b-4ddceb4188d0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6818","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6819","nonce":1670334570}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6820","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6821","nonce":1670334570}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6824","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6825","nonce":1670334570}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.103:6822","nonce":1670334570},{"type":"v1","addr":"10.20.193.103:6823","nonce":1670334570}]},"public_addr":"10.20.193.103:6819/1670334570","cluster_addr":"10.20.193.103:6821/1670334570","heartbeat_back_addr":"10.20.193.103:6825/1670334570","heartbeat_front_addr":"10.20.193.103:6823/1670334570","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:23:48.513215+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:23:47.964110+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:23:48.733361+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"10.20.193.190:0/4064805364":"2026-02-22T23:29:21.066304+0000","10.20.193.190:0/1241769578":"2026-02-22T23:29:17.394270+0000","10.20.193.190:0/165101640":"2026-02-22T23:27:21.619606+0000","10.20.193.190:0/3080161025":"2026-02-22T23:27:21.630782+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:31:34.317 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-22T22:31:34.318 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:31:34.399 INFO:teuthology.orchestra.run.trial103.stdout:{ 2026-02-22T22:31:34.400 INFO:teuthology.orchestra.run.trial103.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:31:34.400 INFO:teuthology.orchestra.run.trial103.stdout:} 2026-02-22T22:31:34.408 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-22T22:31:34.550 INFO:teuthology.orchestra.run.trial103.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T22:31:34.562 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-22T22:31:34.562 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:31:34.639 INFO:teuthology.orchestra.run.trial103.stdout:{ 2026-02-22T22:31:34.639 INFO:teuthology.orchestra.run.trial103.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:31:34.639 INFO:teuthology.orchestra.run.trial103.stdout:} 2026-02-22T22:31:34.648 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-22T22:31:34.819 INFO:teuthology.orchestra.run.trial103.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T22:31:34.829 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-22T22:31:34.829 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:31:34.903 INFO:teuthology.orchestra.run.trial103.stdout:{ 2026-02-22T22:31:34.903 INFO:teuthology.orchestra.run.trial103.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:31:34.903 INFO:teuthology.orchestra.run.trial103.stdout:} 2026-02-22T22:31:34.912 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-22T22:31:35.063 INFO:teuthology.orchestra.run.trial103.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T22:31:35.074 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:31:35.225 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:31:35.226 INFO:teuthology.orchestra.run.trial103.stderr:dumped all 2026-02-22T22:31:35.236 INFO:teuthology.orchestra.run.trial103.stdout:{"pg_ready":true,"pg_map":{"version":303,"stamp":"2026-02-22T22:31:35.170595+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":88032,"kb_used_data":2980,"kb_used_omap":4,"kb_used_meta":84987,"kb_avail":283027488,"statfs":{"total":289910292480,"available":289820147712,"internally_reserved":0,"allocated":3051520,"data_stored":2026630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":87027027},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000929"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":135,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441360+0000","last_change":"2026-02-22T22:23:59.140328+0000","last_active":"2026-02-22T22:31:07.441360+0000","last_peered":"2026-02-22T22:31:07.441360+0000","last_clean":"2026-02-22T22:31:07.441360+0000","last_became_active":"2026-02-22T22:23:57.137067+0000","last_became_peered":"2026-02-22T22:23:57.137067+0000","last_unstale":"2026-02-22T22:31:07.441360+0000","last_undegraded":"2026-02-22T22:31:07.441360+0000","last_fullsized":"2026-02-22T22:31:07.441360+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for deep scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061824999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":135,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441478+0000","last_change":"2026-02-22T22:23:59.140089+0000","last_active":"2026-02-22T22:31:07.441478+0000","last_peered":"2026-02-22T22:31:07.441478+0000","last_clean":"2026-02-22T22:31:07.441478+0000","last_became_active":"2026-02-22T22:23:57.137050+0000","last_became_peered":"2026-02-22T22:23:57.137050+0000","last_unstale":"2026-02-22T22:31:07.441478+0000","last_undegraded":"2026-02-22T22:31:07.441478+0000","last_fullsized":"2026-02-22T22:31:07.441478+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for deep scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041303899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":135,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.437109+0000","last_change":"2026-02-22T22:23:59.140496+0000","last_active":"2026-02-22T22:31:07.437109+0000","last_peered":"2026-02-22T22:31:07.437109+0000","last_clean":"2026-02-22T22:31:07.437109+0000","last_became_active":"2026-02-22T22:23:57.136991+0000","last_became_peered":"2026-02-22T22:23:57.136991+0000","last_unstale":"2026-02-22T22:31:07.437109+0000","last_undegraded":"2026-02-22T22:31:07.437109+0000","last_fullsized":"2026-02-22T22:31:07.437109+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for deep scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078158500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":135,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.440913+0000","last_change":"2026-02-22T22:23:59.140094+0000","last_active":"2026-02-22T22:31:07.440913+0000","last_peered":"2026-02-22T22:31:07.440913+0000","last_clean":"2026-02-22T22:31:07.440913+0000","last_became_active":"2026-02-22T22:23:57.136930+0000","last_became_peered":"2026-02-22T22:23:57.136930+0000","last_unstale":"2026-02-22T22:31:07.440913+0000","last_undegraded":"2026-02-22T22:31:07.440913+0000","last_fullsized":"2026-02-22T22:31:07.440913+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for deep scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040949200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":133,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.138280+0000","last_change":"2026-02-22T22:23:59.138515+0000","last_active":"2026-02-22T22:31:07.138280+0000","last_peered":"2026-02-22T22:31:07.138280+0000","last_clean":"2026-02-22T22:31:07.138280+0000","last_became_active":"2026-02-22T22:23:57.136352+0000","last_became_peered":"2026-02-22T22:23:57.136352+0000","last_unstale":"2026-02-22T22:31:07.138280+0000","last_undegraded":"2026-02-22T22:31:07.138280+0000","last_fullsized":"2026-02-22T22:31:07.138280+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:56.872656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061167800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":132,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.444375+0000","last_change":"2026-02-22T22:23:59.136866+0000","last_active":"2026-02-22T22:31:07.444375+0000","last_peered":"2026-02-22T22:31:07.444375+0000","last_clean":"2026-02-22T22:31:07.444375+0000","last_became_active":"2026-02-22T22:23:57.136381+0000","last_became_peered":"2026-02-22T22:23:57.136381+0000","last_unstale":"2026-02-22T22:31:07.444375+0000","last_undegraded":"2026-02-22T22:31:07.444375+0000","last_fullsized":"2026-02-22T22:31:07.444375+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:47:13.328371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023819299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":132,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.445875+0000","last_change":"2026-02-22T22:23:59.136809+0000","last_active":"2026-02-22T22:31:07.445875+0000","last_peered":"2026-02-22T22:31:07.445875+0000","last_clean":"2026-02-22T22:31:07.445875+0000","last_became_active":"2026-02-22T22:23:57.136128+0000","last_became_peered":"2026-02-22T22:23:57.136128+0000","last_unstale":"2026-02-22T22:31:07.445875+0000","last_undegraded":"2026-02-22T22:31:07.445875+0000","last_fullsized":"2026-02-22T22:31:07.445875+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:33.645236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000213277,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":136,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441703+0000","last_change":"2026-02-22T22:23:59.140296+0000","last_active":"2026-02-22T22:31:07.441703+0000","last_peered":"2026-02-22T22:31:07.441703+0000","last_clean":"2026-02-22T22:31:07.441703+0000","last_became_active":"2026-02-22T22:23:57.136046+0000","last_became_peered":"2026-02-22T22:23:57.136046+0000","last_unstale":"2026-02-22T22:31:07.441703+0000","last_undegraded":"2026-02-22T22:31:07.441703+0000","last_fullsized":"2026-02-22T22:31:07.441703+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:56.125921+0000","last_clean_scrub_stamp":"2026-02-22T22:23:56.125921+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for deep scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000595328,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":226,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:07.441889+0000","last_change":"2026-02-22T22:23:53.494316+0000","last_active":"2026-02-22T22:31:07.441889+0000","last_peered":"2026-02-22T22:31:07.441889+0000","last_clean":"2026-02-22T22:31:07.441889+0000","last_became_active":"2026-02-22T22:23:53.493528+0000","last_became_peered":"2026-02-22T22:23:53.493528+0000","last_unstale":"2026-02-22T22:31:07.441889+0000","last_undegraded":"2026-02-22T22:31:07.441889+0000","last_fullsized":"2026-02-22T22:31:07.441889+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:23:52.116710+0000","last_clean_scrub_stamp":"2026-02-22T22:23:52.116710+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for deep scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705759,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27568,"kb_used_data":604,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344272,"statfs":{"total":96636764160,"available":96608534528,"internally_reserved":0,"allocated":618496,"data_stored":281952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705759,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32312,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339528,"statfs":{"total":96636764160,"available":96603676672,"internally_reserved":0,"allocated":1216512,"data_stored":872339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738465,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28152,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94343688,"statfs":{"total":96636764160,"available":96607936512,"internally_reserved":0,"allocated":1216512,"data_stored":872339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:31:35.237 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T22:23:56.125921+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=23, tm_sec=56, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:35.237 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T22:23:56.125921+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=23, tm_sec=56, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:35.238 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T22:23:56.125921+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=23, tm_sec=56, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:35.238 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T22:23:56.125921+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=23, tm_sec=56, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:35.238 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T22:23:56.125921+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=23, tm_sec=56, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:35.238 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T22:23:56.125921+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=23, tm_sec=56, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:35.238 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T22:23:56.125921+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=23, tm_sec=56, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:35.238 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T22:23:56.125921+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=23, tm_sec=56, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:35.239 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T22:23:52.116710+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=23, tm_sec=52, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=31, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:31:35.239 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T22:31:55.240 DEBUG:teuthology.orchestra.run.trial103:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:31:55.386 INFO:teuthology.orchestra.run.trial103.stdout: 2026-02-22T22:31:55.387 INFO:teuthology.orchestra.run.trial103.stderr:dumped all 2026-02-22T22:31:55.396 INFO:teuthology.orchestra.run.trial103.stdout:{"pg_ready":true,"pg_map":{"version":313,"stamp":"2026-02-22T22:31:55.172237+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":88020,"kb_used_data":2980,"kb_used_omap":4,"kb_used_meta":84987,"kb_avail":283027500,"statfs":{"total":289910292480,"available":289820160000,"internally_reserved":0,"allocated":3051520,"data_stored":2026630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":87027027},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000898"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":143,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:39.458430+0000","last_change":"2026-02-22T22:31:39.458446+0000","last_active":"2026-02-22T22:31:39.458430+0000","last_peered":"2026-02-22T22:31:39.458430+0000","last_clean":"2026-02-22T22:31:39.458430+0000","last_became_active":"2026-02-22T22:23:57.137067+0000","last_became_peered":"2026-02-22T22:23:57.137067+0000","last_unstale":"2026-02-22T22:31:39.458430+0000","last_undegraded":"2026-02-22T22:31:39.458430+0000","last_fullsized":"2026-02-22T22:31:39.458430+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:39.458366+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:39.458366+0000","last_clean_scrub_stamp":"2026-02-22T22:31:39.458366+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:46:06.244319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061824999999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":143,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:40.413331+0000","last_change":"2026-02-22T22:31:40.413347+0000","last_active":"2026-02-22T22:31:40.413331+0000","last_peered":"2026-02-22T22:31:40.413331+0000","last_clean":"2026-02-22T22:31:40.413331+0000","last_became_active":"2026-02-22T22:23:57.137050+0000","last_became_peered":"2026-02-22T22:23:57.137050+0000","last_unstale":"2026-02-22T22:31:40.413331+0000","last_undegraded":"2026-02-22T22:31:40.413331+0000","last_fullsized":"2026-02-22T22:31:40.413331+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:40.413264+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:40.413264+0000","last_clean_scrub_stamp":"2026-02-22T22:31:40.413264+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:54:44.112345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00041303899999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":143,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:38.425797+0000","last_change":"2026-02-22T22:31:38.425843+0000","last_active":"2026-02-22T22:31:38.425797+0000","last_peered":"2026-02-22T22:31:38.425797+0000","last_clean":"2026-02-22T22:31:38.425797+0000","last_became_active":"2026-02-22T22:23:57.136991+0000","last_became_peered":"2026-02-22T22:23:57.136991+0000","last_unstale":"2026-02-22T22:31:38.425797+0000","last_undegraded":"2026-02-22T22:31:38.425797+0000","last_fullsized":"2026-02-22T22:31:38.425797+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:38.425712+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:38.425712+0000","last_clean_scrub_stamp":"2026-02-22T22:31:38.425712+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:33:24.284262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078158500000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":143,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:37.409885+0000","last_change":"2026-02-22T22:31:37.409908+0000","last_active":"2026-02-22T22:31:37.409885+0000","last_peered":"2026-02-22T22:31:37.409885+0000","last_clean":"2026-02-22T22:31:37.409885+0000","last_became_active":"2026-02-22T22:23:57.136930+0000","last_became_peered":"2026-02-22T22:23:57.136930+0000","last_unstale":"2026-02-22T22:31:37.409885+0000","last_undegraded":"2026-02-22T22:31:37.409885+0000","last_fullsized":"2026-02-22T22:31:37.409885+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:37.409788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:37.409788+0000","last_clean_scrub_stamp":"2026-02-22T22:31:37.409788+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:46:52.386256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040949200000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"72'3","reported_seq":143,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:34.564550+0000","last_change":"2026-02-22T22:31:34.564571+0000","last_active":"2026-02-22T22:31:34.564550+0000","last_peered":"2026-02-22T22:31:34.564550+0000","last_clean":"2026-02-22T22:31:34.564550+0000","last_became_active":"2026-02-22T22:23:57.136352+0000","last_became_peered":"2026-02-22T22:23:57.136352+0000","last_unstale":"2026-02-22T22:31:34.564550+0000","last_undegraded":"2026-02-22T22:31:34.564550+0000","last_fullsized":"2026-02-22T22:31:34.564550+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"72'3","last_scrub_stamp":"2026-02-22T22:31:34.564495+0000","last_deep_scrub":"72'3","last_deep_scrub_stamp":"2026-02-22T22:31:34.564495+0000","last_clean_scrub_stamp":"2026-02-22T22:31:34.564495+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:13:06.456684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061167800000000005,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":141,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:36.638333+0000","last_change":"2026-02-22T22:31:36.638369+0000","last_active":"2026-02-22T22:31:36.638333+0000","last_peered":"2026-02-22T22:31:36.638333+0000","last_clean":"2026-02-22T22:31:36.638333+0000","last_became_active":"2026-02-22T22:23:57.136381+0000","last_became_peered":"2026-02-22T22:23:57.136381+0000","last_unstale":"2026-02-22T22:31:36.638333+0000","last_undegraded":"2026-02-22T22:31:36.638333+0000","last_fullsized":"2026-02-22T22:31:36.638333+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:36.638253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:36.638253+0000","last_clean_scrub_stamp":"2026-02-22T22:31:36.638253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:45:40.159597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00023819299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":141,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:35.592038+0000","last_change":"2026-02-22T22:31:35.592055+0000","last_active":"2026-02-22T22:31:35.592038+0000","last_peered":"2026-02-22T22:31:35.592038+0000","last_clean":"2026-02-22T22:31:35.592038+0000","last_became_active":"2026-02-22T22:23:57.136128+0000","last_became_peered":"2026-02-22T22:23:57.136128+0000","last_unstale":"2026-02-22T22:31:35.592038+0000","last_undegraded":"2026-02-22T22:31:35.592038+0000","last_fullsized":"2026-02-22T22:31:35.592038+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:31:35.591964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:31:35.591964+0000","last_clean_scrub_stamp":"2026-02-22T22:31:35.591964+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:43:36.573657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000213277,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"72'2","reported_seq":145,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:36.420327+0000","last_change":"2026-02-22T22:31:36.420354+0000","last_active":"2026-02-22T22:31:36.420327+0000","last_peered":"2026-02-22T22:31:36.420327+0000","last_clean":"2026-02-22T22:31:36.420327+0000","last_became_active":"2026-02-22T22:23:57.136046+0000","last_became_peered":"2026-02-22T22:23:57.136046+0000","last_unstale":"2026-02-22T22:31:36.420327+0000","last_undegraded":"2026-02-22T22:31:36.420327+0000","last_fullsized":"2026-02-22T22:31:36.420327+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"72'2","last_scrub_stamp":"2026-02-22T22:31:36.420267+0000","last_deep_scrub":"72'2","last_deep_scrub_stamp":"2026-02-22T22:31:36.420267+0000","last_clean_scrub_stamp":"2026-02-22T22:31:36.420267+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:02:53.744030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000595328,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"72'74","reported_seq":237,"reported_epoch":72,"state":"active+clean","last_fresh":"2026-02-22T22:31:35.452810+0000","last_change":"2026-02-22T22:31:35.452810+0000","last_active":"2026-02-22T22:31:35.452810+0000","last_peered":"2026-02-22T22:31:35.452810+0000","last_clean":"2026-02-22T22:31:35.452810+0000","last_became_active":"2026-02-22T22:23:53.493528+0000","last_became_peered":"2026-02-22T22:23:53.493528+0000","last_unstale":"2026-02-22T22:31:35.452810+0000","last_undegraded":"2026-02-22T22:31:35.452810+0000","last_fullsized":"2026-02-22T22:31:35.452810+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"72'74","last_scrub_stamp":"2026-02-22T22:31:35.452718+0000","last_deep_scrub":"72'74","last_deep_scrub_stamp":"2026-02-22T22:31:35.452718+0000","last_clean_scrub_stamp":"2026-02-22T22:31:35.452718+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:28:33.754310+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705763,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27564,"kb_used_data":604,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344276,"statfs":{"total":96636764160,"available":96608538624,"internally_reserved":0,"allocated":618496,"data_stored":281952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705763,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32308,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339532,"statfs":{"total":96636764160,"available":96603680768,"internally_reserved":0,"allocated":1216512,"data_stored":872339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":31848903},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738469,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":28148,"kb_used_data":1188,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94343692,"statfs":{"total":96636764160,"available":96607940608,"internally_reserved":0,"allocated":1216512,"data_stored":872339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:31:55.397 DEBUG:teuthology.orchestra.run.trial103:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-22T22:31:55.554 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T22:31:55.555 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T22:31:55.555 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-22T22:31:55.555 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:31:55.642 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-22T22:31:55.642 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-22T22:31:55.642 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:31:55.742 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-22T22:31:55.742 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-22T22:31:55.742 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:31:58.237 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 is failed for ~0s 2026-02-22T22:32:01.844 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-22T22:32:01.844 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T22:32:01.844 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-22T22:32:01.844 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:32:01.875 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-22T22:32:01.875 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T22:32:01.875 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-22T22:32:01.875 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:32:01.931 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-22T22:32:01.931 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T22:32:01.932 DEBUG:teuthology.orchestra.run.trial103:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T22:32:01.985 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial103.front.sepia.ceph.com 2026-02-22T22:32:01.985 DEBUG:teuthology.orchestra.run.trial103:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-22T22:32:02.039 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial103.front.sepia.ceph.com 2026-02-22T22:32:02.039 DEBUG:teuthology.orchestra.run.trial103:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-22T22:32:02.094 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial103.front.sepia.ceph.com 2026-02-22T22:32:02.095 DEBUG:teuthology.orchestra.run.trial103:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-22T22:32:02.146 INFO:tasks.ceph:Archiving mon data... 2026-02-22T22:32:02.147 DEBUG:teuthology.misc:Transferring archived files from trial103:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215/data/cluster1.mon.a.tgz 2026-02-22T22:32:02.147 DEBUG:teuthology.orchestra.run.trial103:> mktemp 2026-02-22T22:32:02.190 INFO:teuthology.orchestra.run.trial103.stdout:/tmp/tmp.6BVpHQuUPQ 2026-02-22T22:32:02.191 DEBUG:teuthology.orchestra.run.trial103:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.6BVpHQuUPQ 2026-02-22T22:32:02.293 DEBUG:teuthology.orchestra.run.trial103:> sudo chmod 0666 /tmp/tmp.6BVpHQuUPQ 2026-02-22T22:32:02.352 DEBUG:teuthology.orchestra.remote:trial103:/tmp/tmp.6BVpHQuUPQ is 392KB 2026-02-22T22:32:02.408 DEBUG:teuthology.orchestra.run.trial103:> rm -fr /tmp/tmp.6BVpHQuUPQ 2026-02-22T22:32:02.411 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T22:32:02.411 DEBUG:teuthology.orchestra.run.trial103:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:32:02.455 DEBUG:teuthology.orchestra.run.trial190:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:32:02.506 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T22:32:02.511 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T22:32:02.511 INFO:tasks.ceph:Archiving crash dumps... 2026-02-22T22:32:02.514 DEBUG:teuthology.misc:Transferring archived files from trial103:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215/remote/trial103/crash 2026-02-22T22:32:02.514 DEBUG:teuthology.orchestra.run.trial103:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T22:32:02.560 DEBUG:teuthology.misc:Transferring archived files from trial190:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215/remote/trial190/crash 2026-02-22T22:32:02.561 DEBUG:teuthology.orchestra.run.trial190:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T22:32:02.570 INFO:tasks.ceph:Compressing logs... 2026-02-22T22:32:02.570 DEBUG:teuthology.orchestra.run.trial103:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:32:02.608 DEBUG:teuthology.orchestra.run.trial190:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:32:02.615 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19504.log 2026-02-22T22:32:02.618 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19114.log 2026-02-22T22:32:02.619 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-22T22:32:02.619 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19504.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19504.log.gz 2026-02-22T22:32:02.619 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19114.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-22T22:32:02.619 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19114.log.gz 2026-02-22T22:32:02.619 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19348.log 2026-02-22T22:32:02.619 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-22T22:32:02.619 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-osd.1.log: /var/log/ceph/cluster1-client.admin.19348.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19348.log.gz 2026-02-22T22:32:02.619 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19198.log 2026-02-22T22:32:02.619 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19044.log 2026-02-22T22:32:02.620 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1.audit.log: /var/log/ceph/cluster1-client.admin.19198.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19198.log.gz 2026-02-22T22:32:02.620 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-22T22:32:02.620 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19044.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19376.log 2026-02-22T22:32:02.620 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19044.log.gz 2026-02-22T22:32:02.620 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-22T22:32:02.620 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19376.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20022.log 2026-02-22T22:32:02.620 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19376.log.gz 2026-02-22T22:32:02.639 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19430.log 2026-02-22T22:32:02.640 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.20022.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19996.log 2026-02-22T22:32:02.640 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20022.log.gz 2026-02-22T22:32:02.640 INFO:teuthology.orchestra.run.trial103.stderr: 92.6% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-22T22:32:02.640 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19814.log 2026-02-22T22:32:02.640 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19430.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19430.log.gz 2026-02-22T22:32:02.640 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16289.log 2026-02-22T22:32:02.640 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19996.log: /var/log/ceph/cluster1-client.admin.19814.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19996.log.gz 2026-02-22T22:32:02.640 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19814.log.gz 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.16289.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16289.log.gz 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19788.log 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16348.log 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20072.log 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19788.log.gz 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr: 86.3% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.16348.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19944.log 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16348.log.gz 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.20072.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19866.log 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20072.log.gz 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19944.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19840.log 2026-02-22T22:32:02.641 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19944.log.gz 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19069.log 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19866.log.gz 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19840.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19530.log 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19840.log.gz 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19918.log 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19069.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19069.log.gz 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19530.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16413.log 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19530.log.gz 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19918.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19280.log 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19918.log.gz 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.16413.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19200.log 2026-02-22T22:32:02.642 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16413.log.gz 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19280.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19280.log.gz 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19094.log 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19199.log 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19200.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19200.log.gz 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19094.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19273.log 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr: 80.5% -- replaced with /var/log/ceph/cluster1-client.admin.19094.log.gz 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19199.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19199.log.gz 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19478.log 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19273.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20048.log 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19273.log.gz 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19478.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19478.log.gz 2026-02-22T22:32:02.643 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19352.log 2026-02-22T22:32:02.644 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.20048.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19970.log 2026-02-22T22:32:02.644 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20048.log.gz 2026-02-22T22:32:02.644 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19352.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-22T22:32:02.644 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19352.log.gz 2026-02-22T22:32:02.644 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19970.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16313.log 2026-02-22T22:32:02.644 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19970.log.gz 2026-02-22T22:32:02.644 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.12553.log 2026-02-22T22:32:02.644 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.16313.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16313.log.gz 2026-02-22T22:32:02.644 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19140.log 2026-02-22T22:32:02.644 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/ceph.tmp-client.admin.12553.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.12553.log.gzgzip 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.16172.log 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19140.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19892.log 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19140.log.gz 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.16172.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16172.log.gz 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19892.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19892.log.gz 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.18983.log 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19762.log 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.18983.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19166.log 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.18983.log.gz 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19762.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19762.log.gz 2026-02-22T22:32:02.645 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19274.log 2026-02-22T22:32:02.646 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19166.log: gzip 0.0% -5 -- replaced with /var/log/ceph/cluster1-client.admin.19166.log.gz --verbose 2026-02-22T22:32:02.646 INFO:teuthology.orchestra.run.trial103.stderr: -- /var/log/ceph/cluster1-client.admin.19426.log 2026-02-22T22:32:02.646 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19274.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19274.log.gz 2026-02-22T22:32:02.646 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.16265.log 2026-02-22T22:32:02.646 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.19426.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19426.log.gz 2026-02-22T22:32:02.646 INFO:teuthology.orchestra.run.trial103.stderr:/var/log/ceph/cluster1-client.admin.16265.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.16265.log.gz 2026-02-22T22:32:02.653 INFO:teuthology.orchestra.run.trial103.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-22T22:32:02.659 INFO:teuthology.orchestra.run.trial103.stderr: 91.8% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33298.log 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29623.log 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33298.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37236.log 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33298.log.gz 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27179.log 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.29623.log: /var/log/ceph/cluster2-client.admin.37236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43173.log 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.37236.log.gz 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.29623.log.gz 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27179.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20515.log 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27179.log.gz 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.admin.43173.log: -- /var/log/ceph/cluster2-client.mirror.28029.log 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43173.log.gz 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20515.log: 65.1% -- replaced with /var/log/ceph/cluster2-client.mirror.20515.log.gz 2026-02-22T22:32:02.669 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27506.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21848.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.28029.log.gz 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27020.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27506.log: /var/log/ceph/cluster2-client.mirror.21848.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19914.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.21848.log.gz 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27020.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32590.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43632.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19914.log: /var/log/ceph/cluster2-client.mirror.32590.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19914.log.gz 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20204.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.27506.log.gz 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43632.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24115.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43632.log.gz 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.20204.log: 2.3% -- replaced with /var/log/ceph/cluster1-client.mirror.20204.log.gz 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30245.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24115.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20136.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.24115.log.gz 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30245.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19174.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.20136.log: 71.2% -- replaced with /var/log/ceph/cluster1-client.admin.20136.log.gz 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43864.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26854.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19174.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.30245.log.gz 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19174.log.gz 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr:gzip/var/log/ceph/cluster2-client.admin.43864.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40135.log 2026-02-22T22:32:02.670 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43864.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.26854.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36077.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.27020.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40135.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32382.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40135.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36077.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31258.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36077.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.32382.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.26854.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40790.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.32382.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31258.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20238.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31258.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40790.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30095.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40790.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.20238.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29163.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr: 58.7% -- replaced with /var/log/ceph/cluster1-client.mirror.20238.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.36372.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.30095.log: /var/log/ceph/cluster2-client.mirror.29163.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31326.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.36372.log: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.29163.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.30095.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25765.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.31326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41784.log 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.31326.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.25765.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.36372.log.gz 2026-02-22T22:32:02.671 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.25222.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.admin.41784.log: /var/log/ceph/cluster2-client.admin.20306.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.admin.25765.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.0.25222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19664.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.25222.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.20306.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.20306.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33708.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19664.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39811.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19664.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38387.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33708.log: /var/log/ceph/cluster2-client.admin.39811.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36042.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38387.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.admin.41784.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.28656.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38387.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: 95.6%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.32590.log.gz -5 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.26109.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36042.log: /var/log/ceph/cluster1-client.mirror.28656.log: gzip -5 --verbose -- 92.3% /var/log/ceph/cluster2-client.admin.16388.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.33708.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36042.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26109.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28689.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: 90.8% -- replaced with /var/log/ceph/cluster1-client.mirror.28656.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.16388.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26109.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29555.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16388.log.gz 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28689.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28266.log 2026-02-22T22:32:02.672 INFO:teuthology.orchestra.run.trial190.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.admin.39811.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37884.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29555.log: /var/log/ceph/cluster2-client.mirror.28266.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29555.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38074.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28266.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.37884.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24723.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.38074.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33055.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.37884.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40192.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24723.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.38074.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.33055.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24723.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39698.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.33055.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40192.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19714.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40192.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31014.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.39698.log: /var/log/ceph/cluster2-client.admin.19714.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19714.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26052.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.39698.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.38854.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31014.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.28689.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26052.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26052.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41030.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31014.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37009.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.38854.log: /var/log/ceph/cluster2-client.admin.41030.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33550.log 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.41030.log.gz 2026-02-22T22:32:02.673 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34799.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.37009.log: gzip/var/log/ceph/cluster1-client.admin.33550.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.26924.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.37009.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34799.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.33550.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.38854.log.gzgzip 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32946.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34799.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.26924.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38320.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35985.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32946.log: /var/log/ceph/cluster2-client.mirror.38320.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.admin.26924.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40268.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32946.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35985.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31476.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40268.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35985.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21924.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40268.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31476.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42085.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19764.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.21924.log: /var/log/ceph/cluster2-client.mirror.42085.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.21924.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26144.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42085.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.31476.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.19764.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21883.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19764.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35666.log 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26144.log.gz 2026-02-22T22:32:02.674 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25197.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.21883.log: /var/log/ceph/cluster2-client.mirror.35666.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19512.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.21883.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35666.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.25197.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30882.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19512.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28849.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19512.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.30882.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.25197.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22887.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28849.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.36406.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28849.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25960.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.22887.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32707.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.3.36406.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.admin.30882.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25960.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19614.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25960.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32707.log: gzip -5 --verbose 94.1% -- -- replaced with /var/log/ceph/cluster2-client.mirror.38320.log.gz /var/log/ceph/cluster2-client.mirror.42727.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr: 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.19614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32487.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42727.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19614.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42727.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41359.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.32487.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38693.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.41359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34205.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41359.log.gz 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.38693.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19206.log 2026-02-22T22:32:02.675 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34205.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.31496.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34205.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19206.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.admin.32487.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23567.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19206.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.31496.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23567.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32707.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35418.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.admin.38693.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.23567.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16319.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35418.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20187.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35418.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.31496.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.16319.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41394.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16319.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.20187.log: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.20187.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.35223.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.41394.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.24818.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41394.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19278.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.0.35223.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.35223.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.24818.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41562.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.24818.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19278.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19278.log.gz 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26967.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33233.log 2026-02-22T22:32:02.676 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.41562.log: /var/log/ceph/cluster2-client.mirror.26967.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35885.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41562.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.33233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28814.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.35885.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35950.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25730.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28814.log: 91.1% -- replaced with /var/log/ceph/cluster2-client.mirror.26967.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39344.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35950.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28814.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.25730.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.21992.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35950.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.39344.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19889.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.33233.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.21992.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.25730.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20153.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.21992.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr: 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.35885.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32866.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19889.log: /var/log/ceph/cluster1-client.mirror.20153.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39198.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19889.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr: 56.2% -- replaced with /var/log/ceph/cluster1-client.mirror.20153.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.33084.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.39198.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32866.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19689.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39198.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.33084.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39598.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19689.log: 92.6% 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19689.log.gz -- replaced with /var/log/ceph/cluster2-client.admin.39344.log.gz 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr: 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30027.log 2026-02-22T22:32:02.677 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.39598.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21675.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.admin.33084.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30027.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37851.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.21675.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30027.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35699.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.21675.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28174.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.37851.log: /var/log/ceph/cluster1-client.admin.35699.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27894.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37851.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.35699.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28174.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31949.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28174.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.27894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19102.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.3.36406.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.27894.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31949.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20481.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31949.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19102.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20532.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 80.7% -- replaced with /var/log/ceph/cluster2-client.admin.19102.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20481.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26987.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 65.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20481.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20532.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.25231.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 54.5% -- replaced with /var/log/ceph/cluster2-client.mirror.20532.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.26987.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.25859.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.0.25231.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20447.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.25859.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.6.36505.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-client.mirror.26987.log.gz 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20447.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35232.log 2026-02-22T22:32:02.678 INFO:teuthology.orchestra.run.trial190.stderr: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.20447.log.gz 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.6.36505.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24975.log 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.6.36505.log.gz 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35232.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24756.log 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.25859.log.gz 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24975.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39727.log 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.24756.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24975.log.gz 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40303.log 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.39598.log.gz 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.24756.log.gz 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.39727.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19839.log 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42050.log 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40303.log.gz 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.19839.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27970.log 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19839.log.gz 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30198.log 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.admin.39727.log.gz 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42050.log.gz 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27970.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24411.log 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.30198.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25043.log 2026-02-22T22:32:02.679 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24411.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41671.log 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.25043.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40758.log 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.27970.log.gz 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.41671.log: gzip -5 88.3% --verbose -- -- replaced with /var/log/ceph/cluster1-client.admin.25043.log.gz /var/log/ceph/cluster2-client.mirror.41841.log 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr: 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.admin.30198.log.gz 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40758.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20119.log 88.2% 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.41671.log.gz 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.41841.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27255.log 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.20119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36264.log 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.admin.20119.log.gz 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.20578.log 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27255.log: /var/log/ceph/cluster2-client.admin.36264.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27255.log.gz 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37816.log 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.36264.log.gz 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:gzip/var/log/ceph/cluster1-client.mirror.0.20578.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.19434.log 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.37816.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24034.log 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.41841.log.gz 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37816.log.gz 2026-02-22T22:32:02.685 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26605.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19434.log: /var/log/ceph/cluster2-client.mirror.24034.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19434.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29307.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24034.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26605.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16436.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29307.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.40758.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26605.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29307.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42879.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30711.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.16436.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16436.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42879.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30467.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42879.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30711.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19739.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30711.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30467.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35777.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.24411.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.19739.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19739.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35865.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30467.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.35777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33482.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.admin.35777.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42635.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 95.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35232.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35631.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33482.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32411.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42635.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33482.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35631.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38520.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35631.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42635.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35865.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.32411.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31838.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38520.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31873.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38520.log.gz 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19462.log 2026-02-22T22:32:02.686 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31838.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31838.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31873.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37913.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32536.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31873.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19462.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.admin.32411.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19462.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.37913.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31429.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32536.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.37465.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.31429.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19814.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25839.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.37465.log: /var/log/ceph/cluster1-client.admin.19814.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19814.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.32536.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43149.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.admin.31429.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29986.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25839.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32257.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.43149.log: /var/log/ceph/cluster2-client.mirror.29986.log: 92.6% -- replaced with /var/log/ceph/cluster1-client.admin.37913.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28209.log 0.0% 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.43149.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.29986.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.37465.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32257.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37964.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32257.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28209.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20221.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28209.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.37964.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.20221.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.25839.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20846.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 65.1% -- replaced with /var/log/ceph/cluster1-client.mirror.20221.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.admin.37964.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41527.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20846.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.35232.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.20846.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.41527.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34732.log 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41527.log.gz 2026-02-22T22:32:02.687 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.0.35232.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25419.log 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23750.log 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34732.log: /var/log/ceph/cluster2-client.mirror.25419.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26716.log 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25419.log.gz 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.23750.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.29149.log 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26716.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35045.log 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.2.29149.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26716.log.gz 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr: 20.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.29149.log.gz 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.23750.log.gz 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24612.log 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41861.log 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.35045.log: /var/log/ceph/cluster2-client.mirror.24612.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20077.log 2026-02-22T22:32:02.688 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24612.log.gz 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.41861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23629.log 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20289.log 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.20077.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20077.log.gz 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.23629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25348.log 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38887.log 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.20289.log: 90.5% -- replaced with /var/log/ceph/cluster1-client.admin.23629.log.gz 92.5% 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.35045.log.gz 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25348.log: 56.3% -- replaced with /var/log/ceph/cluster1-client.mirror.20289.log.gz 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25591.log 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34732.log.gz 2026-02-22T22:32:02.689 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37759.log 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25591.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29275.log 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.41861.log.gz 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25591.log.gz 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.37759.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.37759.log.gz 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32454.log 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37269.log 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29275.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.26335.log 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.32454.log: /var/log/ceph/cluster2-client.admin.37269.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23600.log 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.admin.32454.log.gz 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.22085.log 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.26335.log: /var/log/ceph/cluster1-client.admin.23600.log: gzip 88.3% -5 -- replaced with /var/log/ceph/cluster1-client.admin.23600.log.gz --verbose 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr: -- /var/log/ceph/cluster1-client.admin.28987.log 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.22085.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.admin.37269.log.gz 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31914.log 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.26335.log.gz 2026-02-22T22:32:02.690 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.28987.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42844.log 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.25348.log.gz 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.22085.log.gz 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31914.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22304.log 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31914.log.gz 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38887.log.gz 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42844.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33390.log 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.28987.log.gz 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42844.log.gz 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29059.log 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.22304.log.gz 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30502.log 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33390.log.gz 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.29059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36805.log 2026-02-22T22:32:02.691 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30502.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36215.log 2026-02-22T22:32:02.692 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30502.log.gz 2026-02-22T22:32:02.692 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36805.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25511.log 2026-02-22T22:32:02.692 INFO:teuthology.orchestra.run.trial190.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.admin.29059.log.gz 2026-02-22T22:32:02.692 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30433.log 2026-02-22T22:32:02.692 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.36215.log: /var/log/ceph/cluster2-client.mirror.25511.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20396.log 2026-02-22T22:32:02.692 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25511.log.gz 2026-02-22T22:32:02.693 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30433.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30265.log 2026-02-22T22:32:02.693 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23730.log 2026-02-22T22:32:02.693 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.20396.log: 71.0% -- replaced with /var/log/ceph/cluster2-client.admin.20396.log.gz 2026-02-22T22:32:02.693 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.30265.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.38664.log 2026-02-22T22:32:02.694 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23730.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28591.log 2026-02-22T22:32:02.694 INFO:teuthology.orchestra.run.trial190.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.29275.log.gz 2026-02-22T22:32:02.694 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.30265.log.gz 2026-02-22T22:32:02.694 INFO:teuthology.orchestra.run.trial190.stderr:gzip/var/log/ceph/cluster1-client.admin.38664.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.41751.log 2026-02-22T22:32:02.694 INFO:teuthology.orchestra.run.trial190.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.admin.36215.log.gz 2026-02-22T22:32:02.694 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.28591.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41700.log 2026-02-22T22:32:02.694 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.38664.log.gz 2026-02-22T22:32:02.695 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35178.log 2026-02-22T22:32:02.695 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.41751.log: /var/log/ceph/cluster1-client.admin.41700.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.23730.log.gz 2026-02-22T22:32:02.695 INFO:teuthology.orchestra.run.trial190.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.20578.log.gz 2026-02-22T22:32:02.695 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.28558.log 2026-02-22T22:32:02.695 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.admin.28591.log.gz 2026-02-22T22:32:02.695 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.41751.log.gz 2026-02-22T22:32:02.695 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.21515.log 2026-02-22T22:32:02.696 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35178.log: /var/log/ceph/cluster2-client.admin.28558.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.30949.log 2026-02-22T22:32:02.696 INFO:teuthology.orchestra.run.trial190.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.28558.log.gz 2026-02-22T22:32:02.696 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.1.21515.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39630.log 2026-02-22T22:32:02.696 INFO:teuthology.orchestra.run.trial190.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.21515.log.gz 2026-02-22T22:32:02.696 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.30949.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.admin.41700.log.gz 2026-02-22T22:32:02.696 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26757.log 2026-02-22T22:32:02.697 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.35178.log.gz 2026-02-22T22:32:02.697 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.39630.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.36805.log.gz 2026-02-22T22:32:02.697 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28422.log 2026-02-22T22:32:02.697 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39630.log.gz 2026-02-22T22:32:02.697 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26757.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25072.log 2026-02-22T22:32:02.698 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26757.log.gz 2026-02-22T22:32:02.698 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28422.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.mirror.30949.log.gz 2026-02-22T22:32:02.698 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26681.log 2026-02-22T22:32:02.698 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28422.log.gz 2026-02-22T22:32:02.698 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.25072.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38834.log 2026-02-22T22:32:02.698 INFO:teuthology.orchestra.run.trial190.stderr: 91.4% -- replaced with /var/log/ceph/cluster1-client.admin.25072.log.gz 2026-02-22T22:32:02.698 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26681.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25138.log 2026-02-22T22:32:02.699 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38834.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.26681.log.gz 2026-02-22T22:32:02.699 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26382.log 2026-02-22T22:32:02.699 INFO:teuthology.orchestra.run.trial190.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.30433.log.gz 2026-02-22T22:32:02.699 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.25138.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22760.log 2026-02-22T22:32:02.699 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.22557.log 2026-02-22T22:32:02.700 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26382.log: /var/log/ceph/cluster2-client.mirror.22760.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.38834.log.gz 2026-02-22T22:32:02.700 INFO:teuthology.orchestra.run.trial190.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.26382.log.gz 2026-02-22T22:32:02.700 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22760.log.gz 2026-02-22T22:32:02.700 INFO:teuthology.orchestra.run.trial190.stderr: 91.1% -- replaced with /var/log/ceph/cluster2-client.admin.25138.log.gz 2026-02-22T22:32:02.700 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42768.log 2026-02-22T22:32:02.700 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.29158.log 2026-02-22T22:32:02.700 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.22557.log: /var/log/ceph/cluster2-client.mirror.42768.log: 87.5%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.42768.log.gz 2026-02-22T22:32:02.700 INFO:teuthology.orchestra.run.trial190.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.30779.log 2026-02-22T22:32:02.700 INFO:teuthology.orchestra.run.trial190.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.admin.22557.log.gz 2026-02-22T22:32:02.701 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.2.29158.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.23693.log 2026-02-22T22:32:02.701 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.30779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19207.log 2026-02-22T22:32:02.701 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20255.log/var/log/ceph/cluster2-client.admin.23693.log: 2026-02-22T22:32:02.701 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.30779.log.gz 2026-02-22T22:32:02.702 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19207.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19207.log.gz 2026-02-22T22:32:02.702 INFO:teuthology.orchestra.run.trial190.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.admin.23693.log.gz 2026-02-22T22:32:02.702 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35092.log 2026-02-22T22:32:02.702 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.20255.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32054.log 2026-02-22T22:32:02.702 INFO:teuthology.orchestra.run.trial190.stderr: 65.3% -- replaced with /var/log/ceph/cluster1-client.mirror.20255.log.gz 2026-02-22T22:32:02.702 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.35092.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23999.log 2026-02-22T22:32:02.703 INFO:teuthology.orchestra.run.trial190.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.0.35232.log.gz 2026-02-22T22:32:02.703 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.32054.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23875.log 2026-02-22T22:32:02.703 INFO:teuthology.orchestra.run.trial190.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.admin.35092.log.gz 2026-02-22T22:32:02.703 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36539.log 2026-02-22T22:32:02.703 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.32054.log.gz 2026-02-22T22:32:02.703 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23999.log.gz 2026-02-22T22:32:02.704 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23875.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.37997.log 2026-02-22T22:32:02.704 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36539.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42380.log 2026-02-22T22:32:02.704 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.37997.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33657.log 2026-02-22T22:32:02.705 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42380.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26274.log 2026-02-22T22:32:02.705 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.33657.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39888.log 2026-02-22T22:32:02.705 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.26274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23843.log 2026-02-22T22:32:02.706 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19789.log 2026-02-22T22:32:02.706 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.39888.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20943.log 2026-02-22T22:32:02.706 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23843.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.23843.log.gz/var/log/ceph/cluster1-client.admin.19789.log: 2026-02-22T22:32:02.706 INFO:teuthology.orchestra.run.trial190.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.admin.26274.log.gz 2026-02-22T22:32:02.706 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19789.log.gz 2026-02-22T22:32:02.706 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20272.log 2026-02-22T22:32:02.706 INFO:teuthology.orchestra.run.trial190.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42380.log.gz 2026-02-22T22:32:02.706 INFO:teuthology.orchestra.run.trial190.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.admin.37997.log.gz 2026-02-22T22:32:02.707 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34408.log 2026-02-22T22:32:02.707 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.20943.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.admin.33657.log.gz 2026-02-22T22:32:02.707 INFO:teuthology.orchestra.run.trial190.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.39888.log.gz 2026-02-22T22:32:02.707 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.20272.log: 56.7% -- replaced with /var/log/ceph/cluster1-client.mirror.20272.log.gz 2026-02-22T22:32:02.707 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26017.log 2026-02-22T22:32:02.707 INFO:teuthology.orchestra.run.trial190.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.20943.log.gz 2026-02-22T22:32:02.707 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41326.log 2026-02-22T22:32:02.707 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34408.log: /var/log/ceph/cluster2-client.mirror.26017.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40950.log 2026-02-22T22:32:02.708 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34408.log.gz 2026-02-22T22:32:02.708 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.41326.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.26017.log.gz 2026-02-22T22:32:02.708 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42692.log 2026-02-22T22:32:02.708 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.40950.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.40602.log 2026-02-22T22:32:02.708 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.40950.log.gz 2026-02-22T22:32:02.708 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42692.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37326.log 2026-02-22T22:32:02.709 INFO:teuthology.orchestra.run.trial190.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.23875.log.gz 2026-02-22T22:32:02.709 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42692.log.gz 2026-02-22T22:32:02.709 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.40602.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33129.log 2026-02-22T22:32:02.709 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42323.log 2026-02-22T22:32:02.709 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.37326.log: /var/log/ceph/cluster2-client.admin.33129.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26315.log 2026-02-22T22:32:02.710 INFO:teuthology.orchestra.run.trial190.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.admin.33129.log.gz 2026-02-22T22:32:02.710 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.42323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19564.log 2026-02-22T22:32:02.710 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26315.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32595.log 2026-02-22T22:32:02.710 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42568.log 2026-02-22T22:32:02.711 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19564.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19564.log.gz 2026-02-22T22:32:02.711 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32595.log: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.40602.log.gz 2026-02-22T22:32:02.711 INFO:teuthology.orchestra.run.trial190.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37326.log.gz 2026-02-22T22:32:02.711 INFO:teuthology.orchestra.run.trial190.stderr: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.32595.log.gz 2026-02-22T22:32:02.711 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.32581.log 2026-02-22T22:32:02.711 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42568.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.41326.log.gz 2026-02-22T22:32:02.711 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20914.log 2026-02-22T22:32:02.711 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.26315.log.gz 2026-02-22T22:32:02.712 INFO:teuthology.orchestra.run.trial190.stderr:gzip/var/log/ceph/cluster1-client.mirror.1.32581.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33213.log 2026-02-22T22:32:02.712 INFO:teuthology.orchestra.run.trial190.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.32581.log.gz 2026-02-22T22:32:02.712 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.20914.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22614.log 2026-02-22T22:32:02.712 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.20914.log.gz 2026-02-22T22:32:02.712 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33213.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.admin.42323.log.gz 2026-02-22T22:32:02.712 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30559.log 2026-02-22T22:32:02.712 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.22614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.24319.log 2026-02-22T22:32:02.713 INFO:teuthology.orchestra.run.trial190.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36539.log.gz 2026-02-22T22:32:02.713 INFO:teuthology.orchestra.run.trial190.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.25231.log.gz 2026-02-22T22:32:02.713 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30559.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28381.log 2026-02-22T22:32:02.713 INFO:teuthology.orchestra.run.trial190.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.33213.log.gz 2026-02-22T22:32:02.714 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.24319.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.22052.log 2026-02-22T22:32:02.714 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30559.log.gz 2026-02-22T22:32:02.714 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28381.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20101.log 2026-02-22T22:32:02.714 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28381.log.gz 2026-02-22T22:32:02.714 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.22052.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.admin.24319.log.gz 2026-02-22T22:32:02.714 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30062.log 2026-02-22T22:32:02.714 INFO:teuthology.orchestra.run.trial190.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.admin.22052.log.gz 2026-02-22T22:32:02.714 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.22614.log.gz 2026-02-22T22:32:02.714 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34983.log 2026-02-22T22:32:02.715 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.20101.log: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.20101.log.gz 2026-02-22T22:32:02.715 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30062.log: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.42568.log.gz 2026-02-22T22:32:02.715 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29951.log 2026-02-22T22:32:02.715 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30062.log.gz 2026-02-22T22:32:02.715 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40379.log 2026-02-22T22:32:02.715 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29951.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37044.log 2026-02-22T22:32:02.715 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34983.log.gz 2026-02-22T22:32:02.715 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29951.log.gz 2026-02-22T22:32:02.715 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40441.log 2026-02-22T22:32:02.716 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40379.log: /var/log/ceph/cluster2-client.mirror.37044.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40379.log.gz 2026-02-22T22:32:02.716 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39778.log 2026-02-22T22:32:02.716 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37044.log.gz 2026-02-22T22:32:02.716 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.40441.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28301.log 2026-02-22T22:32:02.716 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32136.log 2026-02-22T22:32:02.717 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.39778.log: /var/log/ceph/cluster2-client.mirror.28301.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28301.log.gz 2026-02-22T22:32:02.717 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24910.log 2026-02-22T22:32:02.717 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.39778.log.gz 2026-02-22T22:32:02.717 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32136.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26177.log 2026-02-22T22:32:02.717 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.24910.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35918.log 2026-02-22T22:32:02.718 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.26177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30670.log 2026-02-22T22:32:02.718 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35918.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.32136.log.gz 2026-02-22T22:32:02.718 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.26177.log.gz 2026-02-22T22:32:02.718 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.22524.log 2026-02-22T22:32:02.718 INFO:teuthology.orchestra.run.trial190.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.admin.40441.log.gz 2026-02-22T22:32:02.718 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30670.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25668.log 2026-02-22T22:32:02.719 INFO:teuthology.orchestra.run.trial190.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.24910.log.gz 2026-02-22T22:32:02.719 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30670.log.gz 2026-02-22T22:32:02.719 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.22524.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22175.log 2026-02-22T22:32:02.719 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25668.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.admin.22524.log.gz 2026-02-22T22:32:02.719 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32831.log 2026-02-22T22:32:02.719 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25668.log.gz 2026-02-22T22:32:02.719 INFO:teuthology.orchestra.run.trial190.stderr:gzip/var/log/ceph/cluster2-client.mirror.22175.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25384.log 2026-02-22T22:32:02.720 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.41140.log 2026-02-22T22:32:02.720 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32831.log: /var/log/ceph/cluster2-client.mirror.25384.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32831.log.gz 2026-02-22T22:32:02.720 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25384.log.gz 2026-02-22T22:32:02.720 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30808.log 2026-02-22T22:32:02.720 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.41140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39122.log 2026-02-22T22:32:02.721 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.30808.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36434.log 2026-02-22T22:32:02.721 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.39122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42600.log 2026-02-22T22:32:02.721 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.39122.log.gz 2026-02-22T22:32:02.721 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34297.log 2026-02-22T22:32:02.722 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36434.log: /var/log/ceph/cluster2-client.mirror.42600.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42600.log.gz 2026-02-22T22:32:02.722 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32156.log 2026-02-22T22:32:02.722 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22493.log 2026-02-22T22:32:02.722 INFO:teuthology.orchestra.run.trial190.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.admin.30808.log.gz 2026-02-22T22:32:02.722 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34297.log: /var/log/ceph/cluster1-client.mirror.32156.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.35918.log.gz 2026-02-22T22:32:02.723 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.41140.log.gz 2026-02-22T22:32:02.723 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34297.log.gz 2026-02-22T22:32:02.723 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43710.log 2026-02-22T22:32:02.723 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.22493.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19864.log 2026-02-22T22:32:02.723 INFO:teuthology.orchestra.run.trial190.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.22175.log.gz 2026-02-22T22:32:02.723 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43710.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.36297.log 2026-02-22T22:32:02.723 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43710.log.gz 2026-02-22T22:32:02.724 INFO:teuthology.orchestra.run.trial190.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.admin.22493.log.gz 2026-02-22T22:32:02.724 INFO:teuthology.orchestra.run.trial190.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.32156.log.gz 2026-02-22T22:32:02.724 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19864.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32349.log 2026-02-22T22:32:02.724 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19864.log.gz 2026-02-22T22:32:02.724 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.36297.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.35198.log 2026-02-22T22:32:02.724 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42142.log 2026-02-22T22:32:02.724 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32349.log.gz 2026-02-22T22:32:02.725 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.35198.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28722.log 2026-02-22T22:32:02.725 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.33728.log 2026-02-22T22:32:02.725 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42142.log: /var/log/ceph/cluster2-client.mirror.28722.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28722.log.gz 2026-02-22T22:32:02.725 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42142.log.gz 2026-02-22T22:32:02.725 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39163.log 2026-02-22T22:32:02.725 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35810.log 2026-02-22T22:32:02.726 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.33728.log: /var/log/ceph/cluster2-client.mirror.39163.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.35198.log.gz 2026-02-22T22:32:02.726 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21524.log 2026-02-22T22:32:02.726 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39163.log.gz 2026-02-22T22:32:02.726 INFO:teuthology.orchestra.run.trial190.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.admin.36297.log.gz 2026-02-22T22:32:02.726 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.35810.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29342.log 2026-02-22T22:32:02.726 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.21524.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27052.log 2026-02-22T22:32:02.727 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29342.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34593.log 2026-02-22T22:32:02.727 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.33728.log.gz 2026-02-22T22:32:02.727 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.29342.log.gz 2026-02-22T22:32:02.727 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27052.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19285.log 2026-02-22T22:32:02.727 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27052.log.gz 2026-02-22T22:32:02.727 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.34593.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23415.log 2026-02-22T22:32:02.728 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19285.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.34593.log.gz 2026-02-22T22:32:02.728 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19285.log.gz 2026-02-22T22:32:02.728 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29773.log 2026-02-22T22:32:02.728 INFO:teuthology.orchestra.run.trial190.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.admin.35810.log.gz 2026-02-22T22:32:02.728 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23415.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32189.log 2026-02-22T22:32:02.728 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29773.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23415.log.gz 2026-02-22T22:32:02.728 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27857.log 2026-02-22T22:32:02.729 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32189.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36434.log.gz 2026-02-22T22:32:02.729 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.12656.log 2026-02-22T22:32:02.729 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25892.log 2026-02-22T22:32:02.729 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.27857.log: /var/log/ceph/ceph.tmp-client.admin.12656.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.12656.log.gz 2026-02-22T22:32:02.729 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19052.log 2026-02-22T22:32:02.730 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25892.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20881.log 2026-02-22T22:32:02.730 INFO:teuthology.orchestra.run.trial190.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.29773.log.gz 2026-02-22T22:32:02.730 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19364.log 2026-02-22T22:32:02.730 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19052.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.admin.27857.log.gz 2026-02-22T22:32:02.730 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19052.log.gz 2026-02-22T22:32:02.730 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20881.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41451.log 2026-02-22T22:32:02.731 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.20881.log.gz 2026-02-22T22:32:02.731 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19364.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19364.log.gz 2026-02-22T22:32:02.731 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38555.log 2026-02-22T22:32:02.731 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22212.log 2026-02-22T22:32:02.731 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.41451.log: /var/log/ceph/cluster2-client.mirror.38555.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38555.log.gz 2026-02-22T22:32:02.731 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26891.log 2026-02-22T22:32:02.731 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.41451.log.gz 2026-02-22T22:32:02.732 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31355.log 2026-02-22T22:32:02.732 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22212.log: gzip/var/log/ceph/cluster2-client.admin.26891.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31797.log 2026-02-22T22:32:02.732 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22212.log.gz 2026-02-22T22:32:02.732 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.31355.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19356.log 2026-02-22T22:32:02.732 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.26891.log.gz 2026-02-22T22:32:02.733 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-22T22:32:02.733 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31797.log.gz 2026-02-22T22:32:02.733 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19356.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30929.log 92.2% 2026-02-22T22:32:02.733 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.31355.log.gz 2026-02-22T22:32:02.733 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19356.log.gz 2026-02-22T22:32:02.733 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35590.log 2026-02-22T22:32:02.734 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30929.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.5.36439.log 2026-02-22T22:32:02.734 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35590.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34373.log 2026-02-22T22:32:02.734 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29693.log 2026-02-22T22:32:02.735 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35590.log.gz 2026-02-22T22:32:02.735 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.5.36439.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.32189.log.gz 2026-02-22T22:32:02.735 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34373.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.5.36439.log.gz 2026-02-22T22:32:02.735 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34373.log.gz 2026-02-22T22:32:02.735 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22795.log 2026-02-22T22:32:02.735 INFO:teuthology.orchestra.run.trial190.stderr: 90.4%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.30929.log.gz 2026-02-22T22:32:02.735 INFO:teuthology.orchestra.run.trial190.stderr: -5 --verbose --/var/log/ceph/cluster2-client.admin.29693.log: /var/log/ceph/cluster2-client.mirror.22339.log 2026-02-22T22:32:02.736 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22795.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.22795.log.gz 2026-02-22T22:32:02.736 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19639.log 2026-02-22T22:32:02.736 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.29693.log.gz 2026-02-22T22:32:02.736 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22339.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.25892.log.gz 2026-02-22T22:32:02.736 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-22T22:32:02.736 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.19639.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22339.log.gz 2026-02-22T22:32:02.736 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19639.log.gz 2026-02-22T22:32:02.736 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.24286.log 2026-02-22T22:32:02.737 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20972.log 2026-02-22T22:32:02.737 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.24286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31705.log 2026-02-22T22:32:02.737 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.24286.log.gz 2026-02-22T22:32:02.737 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.20972.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27526.log 2026-02-22T22:32:02.738 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40582.log 2026-02-22T22:32:02.738 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31705.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.20972.log.gz 2026-02-22T22:32:02.738 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.27526.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31705.log.gz 2026-02-22T22:32:02.738 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32222.log 2026-02-22T22:32:02.738 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40582.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22728.log 2026-02-22T22:32:02.739 INFO:teuthology.orchestra.run.trial190.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.27526.log.gz 2026-02-22T22:32:02.739 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25798.log 2026-02-22T22:32:02.739 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22728.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29894.log 2026-02-22T22:32:02.739 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32222.log.gz 2026-02-22T22:32:02.739 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.25798.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27828.log 2026-02-22T22:32:02.739 INFO:teuthology.orchestra.run.trial190.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.40582.log.gz 2026-02-22T22:32:02.740 INFO:teuthology.orchestra.run.trial190.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.admin.25798.log.gz 2026-02-22T22:32:02.740 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42016.log 2026-02-22T22:32:02.740 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29894.log.gz 2026-02-22T22:32:02.740 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.27828.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41486.log 2026-02-22T22:32:02.740 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.27828.log.gz 2026-02-22T22:32:02.741 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.42016.log: -- /var/log/ceph/cluster2-client.mirror.23456.log 2026-02-22T22:32:02.741 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.41486.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33162.log 2026-02-22T22:32:02.741 INFO:teuthology.orchestra.run.trial190.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.41486.log.gz 2026-02-22T22:32:02.741 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23456.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35728.log 2026-02-22T22:32:02.741 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.33162.log: 87.5%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.23456.log.gz -5 2026-02-22T22:32:02.741 INFO:teuthology.orchestra.run.trial190.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.34484.log 2026-02-22T22:32:02.742 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.35728.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.31396.log 2026-02-22T22:32:02.742 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22963.log 2026-02-22T22:32:02.742 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34484.log: 92.5%/var/log/ceph/cluster2-client.admin.31396.log: -- replaced with /var/log/ceph/cluster2-client.admin.33162.log.gz 2026-02-22T22:32:02.742 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32314.log 2026-02-22T22:32:02.743 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.31396.log.gz 2026-02-22T22:32:02.743 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.34484.log.gz 2026-02-22T22:32:02.743 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30165.log 2026-02-22T22:32:02.743 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22963.log.gz 2026-02-22T22:32:02.743 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32314.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30594.log 2026-02-22T22:32:02.743 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32314.log.gz 2026-02-22T22:32:02.743 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.30165.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36544.log 2026-02-22T22:32:02.743 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.30165.log.gz 2026-02-22T22:32:02.744 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25231.log 2026-02-22T22:32:02.744 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30594.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.admin.35728.log.gz 2026-02-22T22:32:02.744 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30594.log.gz 2026-02-22T22:32:02.744 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36544.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26640.log 2026-02-22T22:32:02.744 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25231.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.36544.log.gz 2026-02-22T22:32:02.745 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19286.log 2026-02-22T22:32:02.745 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23491.log 2026-02-22T22:32:02.745 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26640.log: /var/log/ceph/cluster2-client.admin.19286.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26640.log.gz 2026-02-22T22:32:02.745 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19286.log.gz 2026-02-22T22:32:02.745 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20464.log 2026-02-22T22:32:02.745 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23491.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.22728.log.gz 2026-02-22T22:32:02.745 INFO:teuthology.orchestra.run.trial190.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-client.mirror.21524.log.gz 2026-02-22T22:32:02.745 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23491.log.gz 2026-02-22T22:32:02.745 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26792.log 2026-02-22T22:32:02.746 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27220.log 2026-02-22T22:32:02.746 INFO:teuthology.orchestra.run.trial190.stderr: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.20464.log.gz 2026-02-22T22:32:02.746 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.36186.log 2026-02-22T22:32:02.746 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26792.log: /var/log/ceph/cluster2-client.mirror.27220.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28457.log 2026-02-22T22:32:02.746 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27220.log.gz 2026-02-22T22:32:02.746 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.26792.log.gz 2026-02-22T22:32:02.746 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.36186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29859.log 2026-02-22T22:32:02.747 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28457.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34517.log 2026-02-22T22:32:02.747 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.36186.log.gz 2026-02-22T22:32:02.747 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28457.log.gz 2026-02-22T22:32:02.747 INFO:teuthology.orchestra.run.trial190.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.42016.log.gz 2026-02-22T22:32:02.747 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29859.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21767.log 2026-02-22T22:32:02.747 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.34517.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.29859.log.gz 2026-02-22T22:32:02.747 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30982.log 2026-02-22T22:32:02.748 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.34517.log.gz 2026-02-22T22:32:02.748 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.21767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32739.log 2026-02-22T22:32:02.748 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.21767.log.gz 2026-02-22T22:32:02.748 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30982.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19122.log 2026-02-22T22:32:02.748 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32739.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22594.log 2026-02-22T22:32:02.749 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24890.log 2026-02-22T22:32:02.749 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19122.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32739.log.gz 2026-02-22T22:32:02.749 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22594.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19122.log.gz 2026-02-22T22:32:02.749 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23942.log 2026-02-22T22:32:02.749 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38954.log 2026-02-22T22:32:02.750 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.22594.log.gz 2026-02-22T22:32:02.750 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23942.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19538.log 2026-02-22T22:32:02.750 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38954.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.23942.log.gz 2026-02-22T22:32:02.750 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38954.log.gz 2026-02-22T22:32:02.750 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.33624.log 2026-02-22T22:32:02.751 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19538.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.36397.log 2026-02-22T22:32:02.751 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19538.log.gz 2026-02-22T22:32:02.751 INFO:teuthology.orchestra.run.trial190.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.24890.log.gz 2026-02-22T22:32:02.751 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.33624.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20789.log 2026-02-22T22:32:02.751 INFO:teuthology.orchestra.run.trial190.stderr:gzip/var/log/ceph/cluster1-client.mirror.3.36397.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.38777.log 2026-02-22T22:32:02.751 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.33624.log.gz 2026-02-22T22:32:02.751 INFO:teuthology.orchestra.run.trial190.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.36397.log.gz 2026-02-22T22:32:02.752 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20789.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19964.log 2026-02-22T22:32:02.752 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.20789.log.gz 2026-02-22T22:32:02.752 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.38777.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31141.log 2026-02-22T22:32:02.752 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19964.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25633.log 2026-02-22T22:32:02.752 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19964.log.gz 2026-02-22T22:32:02.753 INFO:teuthology.orchestra.run.trial190.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.29158.log.gz 2026-02-22T22:32:02.753 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31141.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29726.log 2026-02-22T22:32:02.753 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28141.log 2026-02-22T22:32:02.753 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25633.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31141.log.gz 2026-02-22T22:32:02.753 INFO:teuthology.orchestra.run.trial190.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.30982.log.gz 2026-02-22T22:32:02.753 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.29726.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28519.log 2026-02-22T22:32:02.753 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25633.log.gz 2026-02-22T22:32:02.754 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28141.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34764.log 2026-02-22T22:32:02.754 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.admin.29726.log.gz 2026-02-22T22:32:02.754 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27144.log 92.7% 2026-02-22T22:32:02.754 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.38777.log.gz 2026-02-22T22:32:02.754 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.28519.log: /var/log/ceph/cluster2-client.mirror.34764.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43840.log 2026-02-22T22:32:02.754 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34764.log.gz 2026-02-22T22:32:02.755 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27144.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27144.log.gz 2026-02-22T22:32:02.755 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34240.log 2026-02-22T22:32:02.755 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43840.log: gzip 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43840.log.gz 2026-02-22T22:32:02.755 INFO:teuthology.orchestra.run.trial190.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.43102.log 2026-02-22T22:32:02.755 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34240.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43814.log 2026-02-22T22:32:02.755 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34240.log.gz 2026-02-22T22:32:02.755 INFO:teuthology.orchestra.run.trial190.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.28519.log.gz 2026-02-22T22:32:02.755 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.43102.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36837.log 2026-02-22T22:32:02.756 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43814.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43814.log.gz 2026-02-22T22:32:02.756 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.24851.log 2026-02-22T22:32:02.756 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36837.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33775.log 2026-02-22T22:32:02.756 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.24851.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.5.36494.log 2026-02-22T22:32:02.757 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36837.log.gz 2026-02-22T22:32:02.757 INFO:teuthology.orchestra.run.trial103.stderr: 91.2% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-22T22:32:02.757 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33775.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.31982.log 2026-02-22T22:32:02.757 INFO:teuthology.orchestra.run.trial190.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33775.log.gz 2026-02-22T22:32:02.757 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.43102.log.gz 2026-02-22T22:32:02.757 INFO:teuthology.orchestra.run.trial190.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.admin.24851.log.gz 2026-02-22T22:32:02.757 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.5.36494.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29793.log 2026-02-22T22:32:02.758 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.31982.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24536.log 2026-02-22T22:32:02.758 INFO:teuthology.orchestra.run.trial190.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.28141.log.gz 2026-02-22T22:32:02.758 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.29793.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27627.log 2026-02-22T22:32:02.758 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.31982.log.gz 2026-02-22T22:32:02.758 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24080.log 2026-02-22T22:32:02.758 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24536.log: /var/log/ceph/cluster2-client.mirror.27627.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24536.log.gz 2026-02-22T22:32:02.759 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27627.log.gz 2026-02-22T22:32:02.759 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42210.log 2026-02-22T22:32:02.759 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33517.log 2026-02-22T22:32:02.759 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24080.log: /var/log/ceph/cluster1-client.admin.42210.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.42210.log.gz 2026-02-22T22:32:02.759 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.24080.log.gz 2026-02-22T22:32:02.759 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22396.log 2026-02-22T22:32:02.759 INFO:teuthology.orchestra.run.trial190.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.29793.log.gz 2026-02-22T22:32:02.760 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-22T22:32:02.760 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33517.log: 87.5%/var/log/ceph/cluster2-client.mirror.22396.log: -- replaced with /var/log/ceph/cluster2-client.mirror.33517.log.gz 2026-02-22T22:32:02.760 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41638.log 2026-02-22T22:32:02.760 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.22396.log.gz 2026-02-22T22:32:02.760 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42290.log 2026-02-22T22:32:02.761 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.41638.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23532.log 2026-02-22T22:32:02.761 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41638.log.gz 2026-02-22T22:32:02.761 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.42290.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37184.log 2026-02-22T22:32:02.761 INFO:teuthology.orchestra.run.trial190.stderr: 86.2% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-22T22:32:02.761 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23532.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27795.log 2026-02-22T22:32:02.762 INFO:teuthology.orchestra.run.trial190.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.admin.42290.log.gz 2026-02-22T22:32:02.762 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.23532.log.gz 2026-02-22T22:32:02.762 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.37184.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40917.log 2026-02-22T22:32:02.762 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27795.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37087.log 2026-02-22T22:32:02.762 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27795.log.gz 2026-02-22T22:32:02.763 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40917.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28958.log 2026-02-22T22:32:02.763 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40917.log.gz 2026-02-22T22:32:02.763 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.37087.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.4.36434.log 2026-02-22T22:32:02.763 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37087.log.gz 2026-02-22T22:32:02.763 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.28958.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24444.log 2026-02-22T22:32:02.764 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.4.36434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38631.log 2026-02-22T22:32:02.764 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.28958.log.gz 2026-02-22T22:32:02.764 INFO:teuthology.orchestra.run.trial190.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.admin.37184.log.gz 2026-02-22T22:32:02.764 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24444.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29399.log 2026-02-22T22:32:02.764 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24444.log.gz 2026-02-22T22:32:02.764 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38631.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16471.log 2026-02-22T22:32:02.765 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29399.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38631.log.gz 2026-02-22T22:32:02.765 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20754.log 2026-02-22T22:32:02.765 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29399.log.gz 2026-02-22T22:32:02.765 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.16471.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23004.log 2026-02-22T22:32:02.765 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16471.log.gz 2026-02-22T22:32:02.765 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20754.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37122.log 2026-02-22T22:32:02.766 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23004.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41120.log 2026-02-22T22:32:02.766 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.20754.log.gz 2026-02-22T22:32:02.766 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23004.log.gz 2026-02-22T22:32:02.766 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22122.log 2026-02-22T22:32:02.766 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.37122.log: /var/log/ceph/cluster2-client.mirror.41120.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37122.log.gz 2026-02-22T22:32:02.767 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38444.log 2026-02-22T22:32:02.767 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.40492.log 2026-02-22T22:32:02.767 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38444.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29104.log 2026-02-22T22:32:02.767 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38444.log.gz 2026-02-22T22:32:02.767 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.41120.log.gz 2026-02-22T22:32:02.768 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.40492.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35475.log 2026-02-22T22:32:02.768 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.40492.log.gz 2026-02-22T22:32:02.768 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29104.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.23132.log 2026-02-22T22:32:02.768 INFO:teuthology.orchestra.run.trial190.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.22122.log.gz 2026-02-22T22:32:02.768 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19002.log 2026-02-22T22:32:02.769 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35475.log.gz 2026-02-22T22:32:02.769 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.23132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22464.log 2026-02-22T22:32:02.769 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19002.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.23132.log.gz 2026-02-22T22:32:02.769 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.23165.log 0.0% 2026-02-22T22:32:02.769 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.19002.log.gz 2026-02-22T22:32:02.769 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.29104.log.gz 2026-02-22T22:32:02.770 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.22464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19939.log 2026-02-22T22:32:02.770 INFO:teuthology.orchestra.run.trial190.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.5.36494.log.gz 2026-02-22T22:32:02.770 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.23165.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.22464.log.gz 2026-02-22T22:32:02.770 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42177.log 2026-02-22T22:32:02.770 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.32590.log 2026-02-22T22:32:02.770 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19939.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19939.log.gz 2026-02-22T22:32:02.770 INFO:teuthology.orchestra.run.trial190.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.admin.23165.log.gz 2026-02-22T22:32:02.770 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22852.log 2026-02-22T22:32:02.771 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.42177.log.gz 2026-02-22T22:32:02.771 INFO:teuthology.orchestra.run.trial190.stderr:gzip/var/log/ceph/cluster1-client.mirror.1.32590.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20549.log 2026-02-22T22:32:02.771 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31217.log 2026-02-22T22:32:02.771 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22852.log: /var/log/ceph/cluster2-client.mirror.20549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31106.log 2026-02-22T22:32:02.771 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22852.log.gz 2026-02-22T22:32:02.772 INFO:teuthology.orchestra.run.trial190.stderr: 56.3% -- replaced with /var/log/ceph/cluster2-client.mirror.20549.log.gz 2026-02-22T22:32:02.772 INFO:teuthology.orchestra.run.trial190.stderr:gzip/var/log/ceph/cluster2-client.mirror.31217.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.41603.log 2026-02-22T22:32:02.772 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31217.log.gz 2026-02-22T22:32:02.772 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31106.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24253.log 2026-02-22T22:32:02.772 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.41603.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23039.log 2026-02-22T22:32:02.772 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31106.log.gz 2026-02-22T22:32:02.773 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.41603.log.gz 2026-02-22T22:32:02.773 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.24253.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27463.log 2026-02-22T22:32:02.773 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23039.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21641.log 2026-02-22T22:32:02.773 INFO:teuthology.orchestra.run.trial190.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.24253.log.gz 2026-02-22T22:32:02.773 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.27463.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23039.log.gz 2026-02-22T22:32:02.774 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29479.log 2026-02-22T22:32:02.774 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.21641.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.19589.log 2026-02-22T22:32:02.774 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35555.log 2026-02-22T22:32:02.774 INFO:teuthology.orchestra.run.trial190.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.admin.27463.log.gz 2026-02-22T22:32:02.774 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29479.log.gz 2026-02-22T22:32:02.774 INFO:teuthology.orchestra.run.trial190.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25231.log.gz 2026-02-22T22:32:02.775 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.19589.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43554.log 2026-02-22T22:32:02.775 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.19589.log.gz 2026-02-22T22:32:02.775 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.26241.log 2026-02-22T22:32:02.775 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43554.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35555.log.gz 2026-02-22T22:32:02.775 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20430.log 2026-02-22T22:32:02.776 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43554.log.gz 2026-02-22T22:32:02.776 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.26241.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32987.log 2026-02-22T22:32:02.776 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.26241.log.gz 2026-02-22T22:32:02.776 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33425.log 2026-02-22T22:32:02.776 INFO:teuthology.orchestra.run.trial190.stderr: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.20430.log.gz 2026-02-22T22:32:02.776 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32987.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27592.log 2026-02-22T22:32:02.777 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27393.log 2026-02-22T22:32:02.777 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32987.log.gz 2026-02-22T22:32:02.777 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33425.log: /var/log/ceph/cluster2-client.mirror.27592.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.20053.log 2026-02-22T22:32:02.777 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33425.log.gz 2026-02-22T22:32:02.777 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27592.log.gz 2026-02-22T22:32:02.777 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.27393.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.41063.log 2026-02-22T22:32:02.777 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.30849.log 2026-02-22T22:32:02.778 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.20053.log: /var/log/ceph/cluster2-client.admin.41063.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.20053.log.gz 2026-02-22T22:32:02.778 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36494.log 2026-02-22T22:32:02.778 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21710.log 2026-02-22T22:32:02.778 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.30849.log: 91.9% -- replaced with /var/log/ceph/cluster1-client.admin.27393.log.gz 2026-02-22T22:32:02.778 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36494.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27331.log 2026-02-22T22:32:02.778 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.30849.log.gz 2026-02-22T22:32:02.779 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.21710.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.42803.log 2026-02-22T22:32:02.779 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.21710.log.gz 2026-02-22T22:32:02.779 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27331.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20583.log 2026-02-22T22:32:02.779 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27331.log.gz 2026-02-22T22:32:02.779 INFO:teuthology.orchestra.run.trial190.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.admin.41063.log.gz 2026-02-22T22:32:02.779 INFO:teuthology.orchestra.run.trial190.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.21641.log.gz 2026-02-22T22:32:02.779 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16412.log 2026-02-22T22:32:02.779 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.42803.log: /var/log/ceph/cluster2-client.mirror.20583.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.28015.log 2026-02-22T22:32:02.779 INFO:teuthology.orchestra.run.trial190.stderr: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.20583.log.gz 2026-02-22T22:32:02.780 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.16412.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.42803.log.gz 2026-02-22T22:32:02.780 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16412.log.gz 2026-02-22T22:32:02.780 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39011.log 2026-02-22T22:32:02.780 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.1.28015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.21005.log 2026-02-22T22:32:02.780 INFO:teuthology.orchestra.run.trial190.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.28015.log.gz 2026-02-22T22:32:02.780 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.39011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23202.log 2026-02-22T22:32:02.781 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39011.log.gz 2026-02-22T22:32:02.781 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.21005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23255.log 2026-02-22T22:32:02.781 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23202.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.21005.log.gz 2026-02-22T22:32:02.781 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28890.log 2026-02-22T22:32:02.781 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23255.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.4.36411.log 2026-02-22T22:32:02.782 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28890.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.16536.log 2026-02-22T22:32:02.782 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28890.log.gz 2026-02-22T22:32:02.782 INFO:teuthology.orchestra.run.trial190.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.4.36434.log.gz 2026-02-22T22:32:02.782 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22431.log 2026-02-22T22:32:02.782 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.4.36411.log: 91.7% -- replaced with /var/log/ceph/cluster2-client.mirror.23202.log.gz 2026-02-22T22:32:02.782 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.16536.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.16536.log.gz 2026-02-22T22:32:02.783 INFO:teuthology.orchestra.run.trial190.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.4.36411.log.gz 2026-02-22T22:32:02.783 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19148.log 2026-02-22T22:32:02.783 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22431.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.27364.log 2026-02-22T22:32:02.783 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22431.log.gz 2026-02-22T22:32:02.783 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19148.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19438.log 2026-02-22T22:32:02.783 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19148.log.gz 2026-02-22T22:32:02.784 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.27364.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25556.log 2026-02-22T22:32:02.784 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19438.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36153.log 2026-02-22T22:32:02.784 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.27364.log.gz 2026-02-22T22:32:02.784 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19438.log.gz 2026-02-22T22:32:02.784 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25556.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.40979.log 2026-02-22T22:32:02.784 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25556.log.gz 2026-02-22T22:32:02.785 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36153.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26206.log 2026-02-22T22:32:02.785 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36153.log.gz 2026-02-22T22:32:02.785 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.40979.log: gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-22T22:32:02.786 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.26206.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39231.log 2026-02-22T22:32:02.786 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2.audit.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.23255.log.gz 2026-02-22T22:32:02.786 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23288.log 2026-02-22T22:32:02.786 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29590.log/var/log/ceph/cluster1-client.admin.39231.log: 2026-02-22T22:32:02.786 INFO:teuthology.orchestra.run.trial190.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.admin.26206.log.gz 2026-02-22T22:32:02.787 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23288.log: 89.3% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-22T22:32:02.787 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24358.log 2026-02-22T22:32:02.787 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23288.log.gz 2026-02-22T22:32:02.787 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.39231.log.gz 2026-02-22T22:32:02.787 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29590.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.29590.log.gz 2026-02-22T22:32:02.787 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27087.log 2026-02-22T22:32:02.787 INFO:teuthology.orchestra.run.trial190.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.admin.40979.log.gz 2026-02-22T22:32:02.787 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24358.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31293.log 2026-02-22T22:32:02.788 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27087.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33022.log 2026-02-22T22:32:02.788 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.27087.log.gz 2026-02-22T22:32:02.788 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31293.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.22142.log 2026-02-22T22:32:02.788 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31293.log.gz 2026-02-22T22:32:02.788 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33022.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34173.log 2026-02-22T22:32:02.789 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.22142.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.33022.log.gz 2026-02-22T22:32:02.789 INFO:teuthology.orchestra.run.trial190.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24358.log.gz 2026-02-22T22:32:02.789 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36929.log 2026-02-22T22:32:02.789 INFO:teuthology.orchestra.run.trial190.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.32590.log.gz 2026-02-22T22:32:02.789 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43658.log 2026-02-22T22:32:02.789 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34173.log: /var/log/ceph/cluster2-client.mirror.36929.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39046.log 2026-02-22T22:32:02.789 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36929.log.gz 2026-02-22T22:32:02.790 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43658.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43658.log.gz 2026-02-22T22:32:02.790 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42239.log 2026-02-22T22:32:02.790 INFO:teuthology.orchestra.run.trial190.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.22142.log.gz 2026-02-22T22:32:02.790 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.39046.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40344.log 2026-02-22T22:32:02.790 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39046.log.gz 2026-02-22T22:32:02.790 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.42239.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29514.log 2026-02-22T22:32:02.791 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40344.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36494.log.gz 2026-02-22T22:32:02.791 INFO:teuthology.orchestra.run.trial190.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.40344.log.gzgzip 2026-02-22T22:32:02.791 INFO:teuthology.orchestra.run.trial190.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28346.log 2026-02-22T22:32:02.791 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29514.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20358.log 2026-02-22T22:32:02.792 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.29514.log.gz 2026-02-22T22:32:02.792 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28346.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24785.log 2026-02-22T22:32:02.792 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.20358.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.28346.log.gz 2026-02-22T22:32:02.792 INFO:teuthology.orchestra.run.trial190.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.20358.log.gz 2026-02-22T22:32:02.792 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24571.log 2026-02-22T22:32:02.792 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.24785.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43606.log 2026-02-22T22:32:02.792 INFO:teuthology.orchestra.run.trial190.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.admin.42239.log.gz 2026-02-22T22:32:02.793 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37667.log 2026-02-22T22:32:02.793 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24571.log: /var/log/ceph/cluster2-client.admin.43606.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43606.log.gz 2026-02-22T22:32:02.793 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.24571.log.gz 2026-02-22T22:32:02.793 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.24785.log.gz 2026-02-22T22:32:02.793 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25236.log 2026-02-22T22:32:02.793 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25925.log 2026-02-22T22:32:02.794 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.37667.log: /var/log/ceph/cluster2-client.mirror.25236.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.25236.log.gz 2026-02-22T22:32:02.794 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37667.log.gz 2026-02-22T22:32:02.794 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33333.log 2026-02-22T22:32:02.794 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25925.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.21060.log 2026-02-22T22:32:02.794 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25925.log.gz 2026-02-22T22:32:02.794 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33333.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24156.log 2026-02-22T22:32:02.795 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.33333.log.gz 2026-02-22T22:32:02.795 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.21060.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.34173.log.gz 2026-02-22T22:32:02.795 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36118.log 2026-02-22T22:32:02.795 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43762.log 2026-02-22T22:32:02.796 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24156.log: /var/log/ceph/cluster2-client.mirror.36118.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31637.log 2026-02-22T22:32:02.796 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.24156.log.gz 2026-02-22T22:32:02.796 INFO:teuthology.orchestra.run.trial190.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.36118.log.gz 2026-02-22T22:32:02.796 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23907.log 2026-02-22T22:32:02.796 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.21060.log.gz 2026-02-22T22:32:02.796 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43762.log.gz 2026-02-22T22:32:02.796 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31637.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19077.log 2026-02-22T22:32:02.797 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20578.log 2026-02-22T22:32:02.797 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.23907.log.gz 2026-02-22T22:32:02.797 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19077.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.33266.log 2026-02-22T22:32:02.797 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19077.log.gz 2026-02-22T22:32:02.797 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19208.log 2026-02-22T22:32:02.798 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.33266.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29434.log 2026-02-22T22:32:02.798 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19208.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19208.log.gz 2026-02-22T22:32:02.798 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30635.log 2026-02-22T22:32:02.798 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43025.log 2026-02-22T22:32:02.799 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29434.log: /var/log/ceph/cluster2-client.mirror.30635.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.29434.log.gz 2026-02-22T22:32:02.799 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27296.log 2026-02-22T22:32:02.799 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.30635.log.gz 2026-02-22T22:32:02.799 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43025.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34856.log 2026-02-22T22:32:02.800 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27296.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.20569.log 2026-02-22T22:32:02.800 INFO:teuthology.orchestra.run.trial190.stderr: 87.5%/var/log/ceph/cluster2-client.mirror.34856.log: -- replaced with /var/log/ceph/cluster2-client.mirror.27296.log.gz 2026-02-22T22:32:02.800 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.34856.log.gz 2026-02-22T22:32:02.800 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28757.log 2026-02-22T22:32:02.800 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40227.log 2026-02-22T22:32:02.801 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.0.20569.log: /var/log/ceph/cluster2-client.mirror.28757.log: 20.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.20569.log.gz 2026-02-22T22:32:02.801 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.28757.log.gz 2026-02-22T22:32:02.801 INFO:teuthology.orchestra.run.trial190.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.admin.43025.log.gz 2026-02-22T22:32:02.801 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.19363.log 2026-02-22T22:32:02.801 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40227.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20718.log 2026-02-22T22:32:02.801 INFO:teuthology.orchestra.run.trial190.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.31637.log.gz 2026-02-22T22:32:02.801 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40227.log.gz 2026-02-22T22:32:02.802 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.19363.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35349.log 2026-02-22T22:32:02.802 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.19363.log.gz 2026-02-22T22:32:02.802 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20718.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.23222.log 2026-02-22T22:32:02.802 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35349.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.29124.log 2026-02-22T22:32:02.802 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40100.log 2026-02-22T22:32:02.803 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.23222.log: /var/log/ceph/cluster1-client.mirror.29124.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43197.log 2026-02-22T22:32:02.803 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40100.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42912.log 2026-02-22T22:32:02.803 INFO:teuthology.orchestra.run.trial190.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.29124.log.gz 2026-02-22T22:32:02.803 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40100.log.gz 2026-02-22T22:32:02.803 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.43197.log: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.33266.log.gz 2026-02-22T22:32:02.804 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32774.log 2026-02-22T22:32:02.804 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43197.log.gz 2026-02-22T22:32:02.804 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.42912.log: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.23222.log.gz 2026-02-22T22:32:02.804 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.35125.log 2026-02-22T22:32:02.804 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.42912.log.gz 2026-02-22T22:32:02.804 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32774.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.42400.log 2026-02-22T22:32:02.804 INFO:teuthology.orchestra.run.trial190.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.20578.log.gz 2026-02-22T22:32:02.805 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32774.log.gz 2026-02-22T22:32:02.805 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21529.log 2026-02-22T22:32:02.805 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.35125.log: /var/log/ceph/cluster1-client.mirror.42400.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35383.log 2026-02-22T22:32:02.805 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.21529.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.21529.log.gz 2026-02-22T22:32:02.805 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.32011.log 2026-02-22T22:32:02.806 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20332.log 2026-02-22T22:32:02.806 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35383.log: /var/log/ceph/cluster1-client.admin.32011.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.42400.log.gz 2026-02-22T22:32:02.806 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21040.log 2026-02-22T22:32:02.806 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.35383.log.gz 2026-02-22T22:32:02.806 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36352.log 2026-02-22T22:32:02.806 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.20332.log: 0.0%/var/log/ceph/cluster2-client.mirror.21040.log: -- replaced with /var/log/ceph/cluster2-client.admin.20332.log.gz 2026-02-22T22:32:02.807 INFO:teuthology.orchestra.run.trial190.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.admin.35125.log.gz 2026-02-22T22:32:02.807 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36872.log 2026-02-22T22:32:02.807 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36352.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26513.log 2026-02-22T22:32:02.807 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.21040.log.gz 2026-02-22T22:32:02.807 INFO:teuthology.orchestra.run.trial190.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.admin.32011.log.gz 2026-02-22T22:32:02.807 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36872.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.21524.log 2026-02-22T22:32:02.808 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26513.log: 87.5% 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.36872.log.gz 2026-02-22T22:32:02.808 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.20718.log.gz 2026-02-22T22:32:02.808 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37599.log 2026-02-22T22:32:02.808 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26513.log.gz 2026-02-22T22:32:02.808 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26481.log 2026-02-22T22:32:02.808 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.1.21524.log: /var/log/ceph/cluster2-client.mirror.37599.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.36352.log.gz 2026-02-22T22:32:02.808 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43580.log 2026-02-22T22:32:02.809 INFO:teuthology.orchestra.run.trial190.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.35349.log.gz 2026-02-22T22:32:02.809 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26481.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.30746.log 2026-02-22T22:32:02.809 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43580.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42941.log 2026-02-22T22:32:02.809 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43580.log.gz 2026-02-22T22:32:02.809 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.30746.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37632.log 2026-02-22T22:32:02.810 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.42941.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.30746.log.gz 2026-02-22T22:32:02.810 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42992.log 2026-02-22T22:32:02.810 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.37632.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23101.log 2026-02-22T22:32:02.810 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37632.log.gz 2026-02-22T22:32:02.811 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.42992.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20498.log 2026-02-22T22:32:02.811 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.23101.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.42992.log.gz 2026-02-22T22:32:02.811 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40882.log 2026-02-22T22:32:02.811 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20498.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25476.log 2026-02-22T22:32:02.812 INFO:teuthology.orchestra.run.trial190.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.admin.23101.log.gz 2026-02-22T22:32:02.812 INFO:teuthology.orchestra.run.trial190.stderr: 59.9% -- replaced with /var/log/ceph/cluster2-client.mirror.20498.log.gz 2026-02-22T22:32:02.812 INFO:teuthology.orchestra.run.trial190.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.37599.log.gz 2026-02-22T22:32:02.812 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40882.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24191.log 2026-02-22T22:32:02.812 INFO:teuthology.orchestra.run.trial190.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.admin.42941.log.gz 2026-02-22T22:32:02.812 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.40882.log.gz 2026-02-22T22:32:02.812 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25476.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23380.log 2026-02-22T22:32:02.812 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24191.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.25476.log.gz 2026-02-22T22:32:02.812 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.24191.log.gz 2026-02-22T22:32:02.813 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43736.log 2026-02-22T22:32:02.813 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.23380.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39665.log 2026-02-22T22:32:02.813 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.23380.log.gz 2026-02-22T22:32:02.813 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43736.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27927.log 2026-02-22T22:32:02.813 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43736.log.gz 2026-02-22T22:32:02.814 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.39665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27719.log 2026-02-22T22:32:02.814 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39665.log.gz 2026-02-22T22:32:02.814 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.27927.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24479.log 2026-02-22T22:32:02.814 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31182.log 2026-02-22T22:32:02.815 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27719.log.gz 2026-02-22T22:32:02.815 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34948.log 2026-02-22T22:32:02.815 INFO:teuthology.orchestra.run.trial190.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.26481.log.gz 2026-02-22T22:32:02.815 INFO:teuthology.orchestra.run.trial190.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.admin.27927.log.gz 2026-02-22T22:32:02.815 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24479.log.gz 2026-02-22T22:32:02.815 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24943.log 2026-02-22T22:32:02.815 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31182.log: /var/log/ceph/cluster2-client.mirror.34948.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31182.log.gz 2026-02-22T22:32:02.815 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31670.log 2026-02-22T22:32:02.816 INFO:teuthology.orchestra.run.trial190.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.34948.log.gz 2026-02-22T22:32:02.816 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24943.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22928.log 2026-02-22T22:32:02.816 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31670.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.20413.log 2026-02-22T22:32:02.816 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31670.log.gz 2026-02-22T22:32:02.817 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22928.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.39260.log 2026-02-22T22:32:02.817 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.22928.log.gz 2026-02-22T22:32:02.817 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.20413.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.34546.log 2026-02-22T22:32:02.817 INFO:teuthology.orchestra.run.trial190.stderr: 55.8% -- replaced with /var/log/ceph/cluster2-client.mirror.20413.log.gz 2026-02-22T22:32:02.817 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.39260.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29826.log 2026-02-22T22:32:02.818 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.34546.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.23660.log 2026-02-22T22:32:02.818 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35237.log 2026-02-22T22:32:02.818 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.29826.log: /var/log/ceph/cluster2-client.admin.23660.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.35016.log 2026-02-22T22:32:02.819 INFO:teuthology.orchestra.run.trial190.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.admin.23660.log.gz 2026-02-22T22:32:02.819 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35237.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.25105.log 2026-02-22T22:32:02.819 INFO:teuthology.orchestra.run.trial190.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.admin.34546.log.gz 2026-02-22T22:32:02.819 INFO:teuthology.orchestra.run.trial190.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.35237.log.gz 2026-02-22T22:32:02.819 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.35016.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21802.log 2026-02-22T22:32:02.819 INFO:teuthology.orchestra.run.trial190.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.24943.log.gz 2026-02-22T22:32:02.819 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.25105.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.35016.log.gz 2026-02-22T22:32:02.820 INFO:teuthology.orchestra.run.trial190.stderr: 92.7% -- replaced with /var/log/ceph/cluster1-client.admin.39260.log.gz 2026-02-22T22:32:02.820 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27684.log 2026-02-22T22:32:02.820 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.21802.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.25105.log.gz 2026-02-22T22:32:02.820 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34891.log 2026-02-22T22:32:02.820 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.21802.log.gz 2026-02-22T22:32:02.820 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27684.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27684.log.gz 2026-02-22T22:32:02.820 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36964.log 2026-02-22T22:32:02.821 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39087.log 2026-02-22T22:32:02.821 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34891.log: /var/log/ceph/cluster2-client.mirror.36964.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34891.log.gz 2026-02-22T22:32:02.821 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38919.log 2026-02-22T22:32:02.821 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.36964.log.gz 2026-02-22T22:32:02.821 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.39087.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.6.36539.log 2026-02-22T22:32:02.822 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.39087.log.gz 2026-02-22T22:32:02.822 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38919.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34449.log 2026-02-22T22:32:02.822 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38919.log.gz 2026-02-22T22:32:02.822 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.6.36539.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.39311.log 2026-02-22T22:32:02.822 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34449.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.24378.log 2026-02-22T22:32:02.822 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34449.log.gz 2026-02-22T22:32:02.823 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.39311.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.26825.log 2026-02-22T22:32:02.823 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.39311.log.gz 2026-02-22T22:32:02.823 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.24378.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.20376.log 2026-02-22T22:32:02.823 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.26825.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34679.log 2026-02-22T22:32:02.823 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.20376.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39401.log 2026-02-22T22:32:02.824 INFO:teuthology.orchestra.run.trial190.stderr: 80.6% -- replaced with /var/log/ceph/cluster2-client.admin.20376.log.gz 2026-02-22T22:32:02.824 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.26825.log.gz 2026-02-22T22:32:02.824 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34679.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.23072.log 2026-02-22T22:32:02.824 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.39401.log: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.24378.log.gz 2026-02-22T22:32:02.824 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38352.log 2026-02-22T22:32:02.824 INFO:teuthology.orchestra.run.trial190.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.29826.log.gz 2026-02-22T22:32:02.825 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.23072.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.34699.log 2026-02-22T22:32:02.825 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38352.log: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.23072.log.gz 2026-02-22T22:32:02.825 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.24224.log 2026-02-22T22:32:02.825 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38352.log.gz 2026-02-22T22:32:02.825 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.34699.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.34679.log.gz 2026-02-22T22:32:02.826 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28024.log 2026-02-22T22:32:02.826 INFO:teuthology.orchestra.run.trial190.stderr: 91.1% -- replaced with /var/log/ceph/cluster2-client.mirror.39401.log.gz 2026-02-22T22:32:02.826 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.24224.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.38744.log 2026-02-22T22:32:02.826 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28024.log: 88.3%gzip -- replaced with /var/log/ceph/cluster1-client.admin.24224.log.gz -5 2026-02-22T22:32:02.826 INFO:teuthology.orchestra.run.trial190.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.43082.log 2026-02-22T22:32:02.827 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.38744.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40068.log 2026-02-22T22:32:02.827 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.38744.log.gz 2026-02-22T22:32:02.827 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.43082.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.26548.log 2026-02-22T22:32:02.827 INFO:teuthology.orchestra.run.trial190.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.34699.log.gz 2026-02-22T22:32:02.827 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40068.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24647.log 2026-02-22T22:32:02.828 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.26548.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.29158.log 2026-02-22T22:32:02.828 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.34626.log 2026-02-22T22:32:02.828 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.26548.log.gz 2026-02-22T22:32:02.828 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24647.log: /var/log/ceph/cluster2-client.mirror.29158.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.29652.log 2026-02-22T22:32:02.828 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.24647.log.gz 2026-02-22T22:32:02.829 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.34626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.20170.log 2026-02-22T22:32:02.829 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.29652.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.43082.log.gz 2026-02-22T22:32:02.829 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25177.log 2026-02-22T22:32:02.829 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.20170.log: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.20170.log.gz 2026-02-22T22:32:02.829 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.36406.log 2026-02-22T22:32:02.830 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25177.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.admin.29652.log.gz 2026-02-22T22:32:02.830 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.29026.log 2026-02-22T22:32:02.830 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.36406.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.37724.log 2026-02-22T22:32:02.830 INFO:teuthology.orchestra.run.trial190.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.admin.34626.log.gz 2026-02-22T22:32:02.831 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.29026.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.22021.log 2026-02-22T22:32:02.831 INFO:teuthology.orchestra.run.trial190.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.admin.29026.log.gz 2026-02-22T22:32:02.831 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.37724.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43684.log 2026-02-22T22:32:02.831 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.37724.log.gz 2026-02-22T22:32:02.831 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.22021.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.25177.log.gz 2026-02-22T22:32:02.831 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.24688.log 2026-02-22T22:32:02.831 INFO:teuthology.orchestra.run.trial190.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.22021.log.gz 2026-02-22T22:32:02.831 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43684.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43684.log.gz 2026-02-22T22:32:02.832 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.25010.log 2026-02-22T22:32:02.832 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.24688.log: gzip -5 --verbose 87.5% -- -- replaced with /var/log/ceph/cluster2-client.mirror.24688.log.gz 2026-02-22T22:32:02.832 INFO:teuthology.orchestra.run.trial190.stderr: /var/log/ceph/cluster2-client.mirror.31049.log 2026-02-22T22:32:02.832 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28925.log 2026-02-22T22:32:02.833 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.25010.log: /var/log/ceph/cluster2-client.mirror.31049.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.25010.log.gz 2026-02-22T22:32:02.833 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.31049.log.gz /var/log/ceph/cluster2-client.admin.40525.log 2026-02-22T22:32:02.833 INFO:teuthology.orchestra.run.trial190.stderr: 2026-02-22T22:32:02.833 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28925.log: gzip 87.5% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.28925.log.gz --verbose 2026-02-22T22:32:02.833 INFO:teuthology.orchestra.run.trial190.stderr: -- /var/log/ceph/cluster1-client.admin.40412.log 2026-02-22T22:32:02.833 INFO:teuthology.orchestra.run.trial190.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.40068.log.gz 2026-02-22T22:32:02.833 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.40525.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.22247.log 2026-02-22T22:32:02.834 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.40412.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.39421.log 2026-02-22T22:32:02.834 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.40412.log.gz 2026-02-22T22:32:02.834 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.22247.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.28490.log 2026-02-22T22:32:02.834 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.39421.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.22247.log.gz 2026-02-22T22:32:02.835 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43221.log 2026-02-22T22:32:02.835 INFO:teuthology.orchestra.run.trial190.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.admin.40525.log.gz 2026-02-22T22:32:02.835 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.28490.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.32087.log 2026-02-22T22:32:02.835 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.28490.log.gz 2026-02-22T22:32:02.835 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43221.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43221.log.gz 2026-02-22T22:32:02.835 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.27990.log 2026-02-22T22:32:02.835 INFO:teuthology.orchestra.run.trial190.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-client.mirror.39421.log.gz 2026-02-22T22:32:02.836 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.32087.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27760.log 2026-02-22T22:32:02.836 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.27990.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38596.log 2026-02-22T22:32:02.836 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27760.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.27760.log.gz 2026-02-22T22:32:02.836 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.23323.log 2026-02-22T22:32:02.836 INFO:teuthology.orchestra.run.trial190.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.admin.32087.log.gz 2026-02-22T22:32:02.837 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.32556.log 2026-02-22T22:32:02.837 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38596.log: /var/log/ceph/cluster2-client.mirror.23323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.40825.log 2026-02-22T22:32:02.837 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.32556.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.38596.log.gz 92.4% 2026-02-22T22:32:02.837 INFO:teuthology.orchestra.run.trial190.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.27990.log.gz 2026-02-22T22:32:02.837 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.23323.log.gz 2026-02-22T22:32:02.837 INFO:teuthology.orchestra.run.trial190.stderr: 95.2% -- replaced with /var/log/ceph/cluster2-client.mirror.28024.log.gz 2026-02-22T22:32:02.837 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.28024.log 2026-02-22T22:32:02.838 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.40825.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.32911.log 2026-02-22T22:32:02.838 INFO:teuthology.orchestra.run.trial190.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.40825.log.gz 2026-02-22T22:32:02.838 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.32556.log.gz 2026-02-22T22:32:02.838 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.mirror.1.28024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38479.log 2026-02-22T22:32:02.838 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.32911.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.32911.log.gz 2026-02-22T22:32:02.839 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31546.log 2026-02-22T22:32:02.839 INFO:teuthology.orchestra.run.trial190.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.36406.log.gz 2026-02-22T22:32:02.839 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.38054.log 2026-02-22T22:32:02.839 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31546.log: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.31546.log.gz 2026-02-22T22:32:02.839 INFO:teuthology.orchestra.run.trial190.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.38479.log.gz 2026-02-22T22:32:02.839 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.21959.log 2026-02-22T22:32:02.840 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.38054.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.27430.log 2026-02-22T22:32:02.840 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.21959.log: gzip 87.5% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.21959.log.gz --verbose 2026-02-22T22:32:02.840 INFO:teuthology.orchestra.run.trial190.stderr: -- /var/log/ceph/cluster1-client.admin.33579.log 2026-02-22T22:32:02.840 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.38054.log.gz 2026-02-22T22:32:02.840 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.27430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.28636.log 2026-02-22T22:32:02.841 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.33579.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.35510.log 2026-02-22T22:32:02.841 INFO:teuthology.orchestra.run.trial190.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.admin.27430.log.gz 2026-02-22T22:32:02.841 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.28636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.31762.log 2026-02-22T22:32:02.842 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.35510.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.35510.log.gz 2026-02-22T22:32:02.842 INFO:teuthology.orchestra.run.trial190.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.21524.log.gz 2026-02-22T22:32:02.842 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.39868.log 2026-02-22T22:32:02.842 INFO:teuthology.orchestra.run.trial190.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.admin.33579.log.gz 2026-02-22T22:32:02.842 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.25701.log 2026-02-22T22:32:02.842 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.31762.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.6.36539.log.gz 2026-02-22T22:32:02.842 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.39868.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.31762.log.gz 2026-02-22T22:32:02.842 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.34332.log 2026-02-22T22:32:02.842 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.25701.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.37155.log 2026-02-22T22:32:02.842 INFO:teuthology.orchestra.run.trial190.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.admin.25701.log.gz 2026-02-22T22:32:02.843 INFO:teuthology.orchestra.run.trial190.stderr: 90.9% -- replaced with /var/log/ceph/cluster2-client.mirror.28636.log.gz 2026-02-22T22:32:02.843 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.34332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43788.log 2026-02-22T22:32:02.843 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster1-client.admin.37155.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.34332.log.gz 2026-02-22T22:32:02.843 INFO:teuthology.orchestra.run.trial190.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.39868.log.gz 2026-02-22T22:32:02.843 INFO:teuthology.orchestra.run.trial190.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.37155.log.gz 2026-02-22T22:32:02.843 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.27559.log 2026-02-22T22:32:02.843 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.admin.43788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.30124.log 2026-02-22T22:32:02.844 INFO:teuthology.orchestra.run.trial190.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43788.log.gz 2026-02-22T22:32:02.846 INFO:teuthology.orchestra.run.trial190.stderr:/var/log/ceph/cluster2-client.mirror.27559.log: /var/log/ceph/cluster1-client.admin.30124.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.admin.30124.log.gz 2026-02-22T22:32:02.850 INFO:teuthology.orchestra.run.trial190.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.27559.log.gz 2026-02-22T22:32:02.856 INFO:teuthology.orchestra.run.trial190.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.28024.log.gz 2026-02-22T22:32:02.857 INFO:teuthology.orchestra.run.trial190.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-22T22:32:02.868 INFO:teuthology.orchestra.run.trial190.stderr: 94.8% -- replaced with /var/log/ceph/cluster2-client.mirror.29158.log.gz 2026-02-22T22:32:03.238 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 has been restored 2026-02-22T22:32:03.633 INFO:teuthology.orchestra.run.trial103.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-22T22:32:03.636 INFO:teuthology.orchestra.run.trial190.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-22T22:32:03.655 INFO:teuthology.orchestra.run.trial103.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-22T22:32:03.664 INFO:teuthology.orchestra.run.trial103.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-22T22:32:03.665 INFO:teuthology.orchestra.run.trial103.stderr: 2026-02-22T22:32:03.665 INFO:teuthology.orchestra.run.trial103.stderr:real 0m1.056s 2026-02-22T22:32:03.665 INFO:teuthology.orchestra.run.trial103.stderr:user 0m3.185s 2026-02-22T22:32:03.665 INFO:teuthology.orchestra.run.trial103.stderr:sys 0m0.155s 2026-02-22T22:32:03.855 INFO:teuthology.orchestra.run.trial190.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-22T22:32:03.908 INFO:teuthology.orchestra.run.trial190.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-22T22:32:03.909 INFO:teuthology.orchestra.run.trial190.stderr: 2026-02-22T22:32:03.909 INFO:teuthology.orchestra.run.trial190.stderr:real 0m1.291s 2026-02-22T22:32:03.909 INFO:teuthology.orchestra.run.trial190.stderr:user 0m4.425s 2026-02-22T22:32:03.909 INFO:teuthology.orchestra.run.trial190.stderr:sys 0m0.433s 2026-02-22T22:32:03.911 INFO:tasks.ceph:Archiving logs... 2026-02-22T22:32:03.911 DEBUG:teuthology.misc:Transferring archived files from trial103:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215/remote/trial103/log 2026-02-22T22:32:03.912 DEBUG:teuthology.orchestra.run.trial103:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T22:32:04.247 DEBUG:teuthology.misc:Transferring archived files from trial190:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215/remote/trial190/log 2026-02-22T22:32:04.249 DEBUG:teuthology.orchestra.run.trial190:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T22:32:04.898 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-22T22:32:04.927 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-22T22:32:04.928 DEBUG:teuthology.orchestra.run.trial103:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T22:32:04.930 DEBUG:teuthology.orchestra.run.trial190:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T22:32:05.072 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-22T22:32:05.084 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-22T22:32:05.084 DEBUG:teuthology.orchestra.run.trial103:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:32:05.085 DEBUG:teuthology.orchestra.run.trial190:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:32:05.108 INFO:teuthology.orchestra.run.trial103.stdout: remote refid st t when poll reach delay offset jitter 2026-02-22T22:32:05.108 INFO:teuthology.orchestra.run.trial103.stdout:============================================================================== 2026-02-22T22:32:05.108 INFO:teuthology.orchestra.run.trial103.stdout:*clock.nyc.he.ne 66.220.9.122 3 u 23 64 377 3.083 +5.177 6.061 2026-02-22T22:32:05.108 INFO:teuthology.orchestra.run.trial103.stdout:+170-187-147-56. 80.192.165.246 2 u 15 64 377 19.745 +9.553 4.129 2026-02-22T22:32:05.108 INFO:teuthology.orchestra.run.trial103.stdout:-172-234-44-141. 81.104.22.229 2 u 24 64 377 7.587 -17.382 2.683 2026-02-22T22:32:05.108 INFO:teuthology.orchestra.run.trial103.stdout:+149.28.61.105 ( 45.63.54.13 3 u 23 64 377 3.364 -1.584 2.350 2026-02-22T22:32:05.118 INFO:teuthology.orchestra.run.trial190.stdout: remote refid st t when poll reach delay offset jitter 2026-02-22T22:32:05.118 INFO:teuthology.orchestra.run.trial190.stdout:============================================================================== 2026-02-22T22:32:05.118 INFO:teuthology.orchestra.run.trial190.stdout:+149.248.12.167 23.95.49.216 3 u 18 64 377 61.888 +3.109 2.811 2026-02-22T22:32:05.118 INFO:teuthology.orchestra.run.trial190.stdout:*europa.ellipse. 69.89.207.99 2 u 20 64 377 31.196 +0.984 1.497 2026-02-22T22:32:05.118 INFO:teuthology.orchestra.run.trial190.stdout:+ntp.maxhost.io 132.163.97.4 2 u 24 64 377 69.631 +0.121 1.271 2026-02-22T22:32:05.118 INFO:teuthology.orchestra.run.trial190.stdout:-172-233-177-198 80.192.165.246 2 u 14 64 377 36.855 +5.803 1.100 2026-02-22T22:32:05.119 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-22T22:32:05.130 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-22T22:32:05.130 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-22T22:32:05.143 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-22T22:32:05.157 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-22T22:32:05.172 INFO:teuthology.task.internal:Duration was 700.284140 seconds 2026-02-22T22:32:05.172 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-22T22:32:05.188 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-22T22:32:05.188 DEBUG:teuthology.orchestra.run.trial103:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T22:32:05.190 DEBUG:teuthology.orchestra.run.trial190:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T22:32:05.215 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-22T22:32:05.215 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial103.front.sepia.ceph.com 2026-02-22T22:32:05.215 DEBUG:teuthology.orchestra.run.trial103:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T22:32:05.261 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial190.front.sepia.ceph.com 2026-02-22T22:32:05.261 DEBUG:teuthology.orchestra.run.trial190:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T22:32:05.268 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-22T22:32:05.268 DEBUG:teuthology.orchestra.run.trial103:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:32:05.304 DEBUG:teuthology.orchestra.run.trial190:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:32:05.541 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-22T22:32:05.541 DEBUG:teuthology.orchestra.run.trial103:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:32:05.543 DEBUG:teuthology.orchestra.run.trial190:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:32:05.551 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:32:05.551 INFO:teuthology.orchestra.run.trial103.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:32:05.553 INFO:teuthology.orchestra.run.trial103.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:32:05.553 INFO:teuthology.orchestra.run.trial103.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T22:32:05.553 INFO:teuthology.orchestra.run.trial103.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: /home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T22:32:05.555 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:32:05.556 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:32:05.556 INFO:teuthology.orchestra.run.trial190.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T22:32:05.556 INFO:teuthology.orchestra.run.trial190.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:32:05.556 INFO:teuthology.orchestra.run.trial190.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: /home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T22:32:05.621 INFO:teuthology.orchestra.run.trial103.stderr: 81.4% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T22:32:05.624 INFO:teuthology.orchestra.run.trial190.stderr: 81.5% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T22:32:05.626 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-22T22:32:05.643 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-22T22:32:05.643 DEBUG:teuthology.orchestra.run.trial103:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T22:32:05.675 DEBUG:teuthology.orchestra.run.trial190:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T22:32:05.684 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-22T22:32:05.702 DEBUG:teuthology.orchestra.run.trial103:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:32:05.720 DEBUG:teuthology.orchestra.run.trial190:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:32:05.726 INFO:teuthology.orchestra.run.trial103.stdout:kernel.core_pattern = core 2026-02-22T22:32:05.735 INFO:teuthology.orchestra.run.trial190.stdout:kernel.core_pattern = core 2026-02-22T22:32:05.744 DEBUG:teuthology.orchestra.run.trial103:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:32:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:32:05.779 DEBUG:teuthology.orchestra.run.trial190:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:32:05.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:32:05.791 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-22T22:32:05.809 INFO:teuthology.task.internal:Transferring archived files... 2026-02-22T22:32:05.810 DEBUG:teuthology.misc:Transferring archived files from trial103:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215/remote/trial103 2026-02-22T22:32:05.810 DEBUG:teuthology.orchestra.run.trial103:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T22:32:05.881 DEBUG:teuthology.misc:Transferring archived files from trial190:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64215/remote/trial190 2026-02-22T22:32:05.881 DEBUG:teuthology.orchestra.run.trial190:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T22:32:05.935 INFO:teuthology.task.internal:Removing archive directory... 2026-02-22T22:32:05.935 DEBUG:teuthology.orchestra.run.trial103:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T22:32:05.937 DEBUG:teuthology.orchestra.run.trial190:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T22:32:05.967 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-22T22:32:05.981 INFO:teuthology.task.internal:Not uploading archives. 2026-02-22T22:32:05.981 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-22T22:32:06.002 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-22T22:32:06.002 DEBUG:teuthology.orchestra.run.trial103:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T22:32:06.004 DEBUG:teuthology.orchestra.run.trial190:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T22:32:06.006 INFO:teuthology.orchestra.run.trial103.stdout: 1179659 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 22:32 /home/ubuntu/cephtest 2026-02-22T22:32:06.011 INFO:teuthology.orchestra.run.trial190.stdout: 1179659 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 22:32 /home/ubuntu/cephtest 2026-02-22T22:32:06.012 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-22T22:32:06.028 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-22T22:32:06.061 INFO:teuthology.run:Summary data: description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-hybrid supported-random-distro$/{ubuntu_latest} workloads/rbd-mirror-ha-workunit} duration: 700.2841403484344 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-22T22:32:06.061 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T22:32:06.168 INFO:teuthology.run:pass